refactor(alloy): drop privileged and proxy the docker socket read-only

Replace privileged: true with cap_drop ALL plus DAC_OVERRIDE, no-new-privileges,
a read-only rootfs and memory/pid limits. DAC_OVERRIDE is what lets the uid-0
entrypoint create its storage directory and read the journal and /rootfs; every
other capability stays dropped.

Route prometheus.exporter.cadvisor, discovery.docker and loki.source.docker
through a docker-socket-proxy sidecar on 127.0.0.1:2375 instead of bind-mounting
the socket. POST is refused there, so container create and exec are no longer
reachable. NETWORKS is granted because Docker SD resolves network names per
container and returns no targets without it.

Add an alloy validate step to CI and boot the test container with the shipped
capability set, read-only rootfs and proxy rather than --privileged.

alloy/.github/workflows/validate.yml

@@ -35,31 +35,50 @@ jobs:
           docker run --rm -v "$PWD/config.alloy:/config.alloy:ro" \
             grafana/alloy:v1.16.1 fmt /config.alloy > /dev/null
 
+      - name: Validate Alloy config arguments
+        run: |
+          docker run --rm \
+            -e ALLOY_HOSTNAME -e REMOTECFG_URL -e REMOTECFG_ID -e REMOTECFG_USER \
+            -e PROM_URL -e PROM_USER -e LOKI_URL -e LOKI_USER -e GRAFANA_TOKEN \
+            -v "$PWD/config.alloy:/config.alloy:ro" \
+            grafana/alloy:v1.16.1 validate /config.alloy
+
       - name: Validate Alloy config semantics (parse + load components)
-        # `fmt` is syntax-only; this catches bad component refs, wrong arg
-        # types, and unknown component types by booting alloy and reading
-        # the load log. Component runtime errors against dummy paths /
+        # `fmt` and `validate` are static; this boots alloy with the same
+        # capability set, read-only rootfs and socket proxy the compose file
+        # uses, and reads the load log. Component runtime errors against dummy paths /
         # endpoints are tolerated (level=error, level=warn) — only fail on
         # config-level errors that prevent the graph from being built.
         run: |
           # /var/log/journal may not exist on the GHA runner; create empty
           # dir so loki.source.journal can at least open it.
           mkdir -p /tmp/empty-journal
-          docker run -d --name alloy-test --privileged \
+          docker volume create alloy-test-data >/dev/null
+          docker run -d --name alloy-test-proxy \
+            --security-opt no-new-privileges:true \
+            -e CONTAINERS=1 -e NETWORKS=1 -e IMAGES=1 -e INFO=1 -e VERSION=1 -e EVENTS=1 \
+            -p 127.0.0.1:2375:2375 \
+            -v /var/run/docker.sock:/var/run/docker.sock:ro \
+            tecnativa/docker-socket-proxy:v0.5.0
+          docker run -d --name alloy-test --network host \
+            --cap-drop ALL --cap-add DAC_OVERRIDE \
+            --security-opt no-new-privileges:true --read-only --tmpfs /tmp \
             -e ALLOY_HOSTNAME -e REMOTECFG_URL -e REMOTECFG_ID -e REMOTECFG_USER \
             -e PROM_URL -e PROM_USER -e LOKI_URL -e LOKI_USER -e GRAFANA_TOKEN \
+            -v alloy-test-data:/var/lib/alloy/data \
             -v "$PWD/config.alloy:/etc/alloy/config.alloy:ro" \
             -v /proc:/rootproc:ro -v /sys:/sys:ro -v /:/rootfs:ro \
-            -v /var/run/docker.sock:/var/run/docker.sock \
             -v /var/lib/docker:/var/lib/docker:ro \
             -v /tmp/empty-journal:/var/log/journal:ro \
             -v /etc/machine-id:/etc/machine-id:ro \
             grafana/alloy:v1.16.1 \
-            run --server.http.listen-addr=127.0.0.1:12345 --disable-reporting /etc/alloy/config.alloy
+            run --server.http.listen-addr=127.0.0.1:12346 --disable-reporting \
+                --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
           sleep 10
           docker logs alloy-test 2>&1 | tee /tmp/alloy.log
           state=$(docker inspect -f '{{.State.Status}}' alloy-test 2>/dev/null || echo missing)
-          docker rm -f alloy-test >/dev/null 2>&1 || true
+          docker rm -f alloy-test alloy-test-proxy >/dev/null 2>&1 || true
+          docker volume rm alloy-test-data >/dev/null 2>&1 || true
           if grep -Eq "could not (parse|load|build)|unknown component|component .* not registered|undefined reference|syntax error|expected .* found|cannot evaluate" /tmp/alloy.log; then
             echo "::error::config-level error detected in alloy logs"
             exit 1

alloy/README.md

@@ -5,7 +5,8 @@ container telemetry to Grafana Cloud, with remote config from Grafana Fleet
 Management.
 
 One container runs both the `node_exporter` (host) and `cadvisor` (container)
-collectors. The Alloy config is embedded inline via Compose `configs:`, so
+collectors. A second, tiny container proxies a read-only slice of the Docker
+API to it. The Alloy config is embedded inline via Compose `configs:`, so
 there is no `config.alloy` on disk, and every setting comes from a shell
 variable rather than a `.env` file.
 
@@ -62,11 +63,59 @@ per host. Filter in Grafana with `instance=~"..."`.
 
 ## Privileges
 
-Runs `privileged: true` with `network_mode: host`, matching the upstream
-Grafana Cloud docker integration. Host networking is what lets
-`prometheus.exporter.unix` report the host's real interfaces (`eth0`…) instead
-of the container's veth pair. To tighten this, see the upstream Alloy docker
-integration docs.
+The upstream Grafana Cloud docker integration runs `privileged: true` with the
+raw Docker socket mounted read-write. Nothing in this config needs that, and
+the combination is host-root-equivalent: `privileged` grants every capability
+and unmasks `/proc` and `/sys`, and a container that can talk to the Docker
+socket can start another container that mounts `/` writable. The socket's `:ro`
+flag does not help — it stops the socket *file* being replaced, not the API
+being used.
+
+What it runs instead:
+
+| Setting | Reason |
+| --- | --- |
+| `cap_drop: [ALL]` + `cap_add: [DAC_OVERRIDE]` | The image's entrypoint runs as uid 0 and reads host files owned by other users — the journal, paths under `/rootfs`, `/var/log`. Dropping every capability leaves it unable to open them, and unable to create its own storage directory. `DAC_OVERRIDE` restores exactly that and nothing else; `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `MKNOD` and the rest stay dropped. |
+| `no-new-privileges:true` | No setuid binary in the image can regain what was dropped. |
+| `read_only: true` with `tmpfs: /tmp` | Only the `alloy-data` volume is writable. |
+| `mem_limit: 2g`, `pids_limit: 512` | Steady state is around 900 MB; the limit stops a leak taking the host down with it. |
+| `dockerproxy` instead of `/var/run/docker.sock` | See below. |
+
+`network_mode: host` stays. `/proc/net` is a symlink to `/proc/self/net` and
+resolves against the reading process's network namespace, so bind-mounting the
+host's `/proc` to `/rootproc` is not enough — without host networking the
+netdev, netstat, sockstat and conntrack families would describe the container's
+veth pair instead of `eth0`. That is roughly 45 of the 157 kept metrics.
+
+`/:/rootfs:ro` also stays, and is the widest remaining exposure: the container
+can read every file on the host. The filesystem collector needs to `statfs()`
+each mount point, and a bind mount cannot grant that without granting reads.
+Dropping it would cost disk-space monitoring. Treat the container as
+secret-bearing — it holds `GRAFANA_TOKEN` regardless.
+
+## Docker API access
+
+`prometheus.exporter.cadvisor` and `loki.source.docker` both need the Docker
+API, so it cannot simply be removed. `dockerproxy` runs
+[tecnativa/docker-socket-proxy](https://github.com/Tecnativa/docker-socket-proxy)
+with the socket mounted read-only and exposes it on `127.0.0.1:2375`, which
+Alloy reaches over host networking.
+
+`POST` is revoked by default in that image, which is the point: container
+create, `exec`, start and kill are all refused with 403, so a compromise of
+Alloy can no longer become root on the host. The granted sections are the ones
+the two components actually call:
+
+| Variable | Called by |
+| --- | --- |
+| `CONTAINERS` | `discovery.docker` listing, `loki.source.docker` inspect and log read, cadvisor metadata |
+| `NETWORKS` | `discovery.docker` — Prometheus' Docker SD resolves network names per container, and returns **zero targets** without it |
+| `IMAGES`, `INFO`, `VERSION` | cadvisor |
+| `EVENTS` | cadvisor container watch (granted by default) |
+
+What this does not do: `GET /containers/{id}/json` still returns every
+container's environment variables. The proxy closes the escalation path, not
+the disclosure one.
 
 ## Mounts
 
@@ -76,11 +125,12 @@ integration docs.
 | `/sys:/sys:ro` | node-exporter and cadvisor cgroups |
 | `/:/rootfs:ro` | filesystem collector, via `rootfs_path` |
 | `/dev/disk/:/dev/disk:ro` | node-exporter diskstats device labels |
-| `/var/run/docker.sock` | `discovery.docker` and `loki.source.docker` |
 | `/var/lib/docker:ro` | cadvisor container metadata |
 | `/var/log:/var/log:ro` | `loki.source.journal` and `loki.source.file` |
 | `/etc/machine-id:ro` | Stable host id for the journal reader |
-| `alloy-data` | WAL and remotecfg cache |
+| `alloy-data` | WAL and remotecfg cache — the only writable path |
+
+`dockerproxy` mounts `/var/run/docker.sock:ro` and nothing else.
 
 ## Notes
 

alloy/docker-compose.yml

@@ -8,8 +8,18 @@ services:
     container_name: alloy
     restart: unless-stopped
     hostname: ${ALLOY_HOSTNAME:?required}
-    privileged: true
     network_mode: host                        # node_exporter netdev/netstat sees real host interfaces (eth0…) not veth
+    cap_drop: [ALL]
+    cap_add: [DAC_OVERRIDE]                   # read host files owned by other uids: journal, /rootfs, /var/log
+    security_opt:
+      - no-new-privileges:true
+    read_only: true
+    tmpfs:
+      - /tmp
+    mem_limit: 2g
+    pids_limit: 512
+    depends_on:
+      - dockerproxy
     environment:
       ALLOY_DEPLOY_MODE: docker
       REMOTECFG_URL: ${REMOTECFG_URL:?required}
@@ -23,7 +33,6 @@ services:
     volumes:
       - alloy-data:/var/lib/alloy/data
       - /proc:/rootproc:ro
-      - /var/run/docker.sock:/var/run/docker.sock
       - /sys:/sys:ro
       - /:/rootfs:ro
       - /dev/disk/:/dev/disk:ro
@@ -34,6 +43,26 @@ services:
       - { source: alloy_config, target: /etc/alloy/config.alloy }
     command: 'run --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy'
 
+  dockerproxy:
+    image: tecnativa/docker-socket-proxy:v0.5.0
+    container_name: alloy-dockerproxy
+    restart: unless-stopped
+    security_opt:
+      - no-new-privileges:true
+    mem_limit: 64m
+    pids_limit: 64
+    environment:                              # the API sections cadvisor and the log tailer read; POST stays revoked
+      CONTAINERS: 1
+      NETWORKS: 1
+      IMAGES: 1
+      INFO: 1
+      VERSION: 1
+      EVENTS: 1
+    ports:
+      - 127.0.0.1:2375:2375
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+
 volumes:
   alloy-data:
 
@@ -195,6 +224,7 @@ configs:
 
       prometheus.exporter.cadvisor "integrations_cadvisor" {
           docker_only = true
+          docker_host = "tcp://127.0.0.1:2375"
       }
       discovery.relabel "integrations_cadvisor" {
           targets = prometheus.exporter.cadvisor.integrations_cadvisor.targets
@@ -226,7 +256,7 @@ configs:
       }
 
       discovery.docker "logs_integrations_docker" {
-          host             = "unix:///var/run/docker.sock"
+          host             = "tcp://127.0.0.1:2375"
           refresh_interval = "5s"
       }
       discovery.relabel "logs_integrations_docker" {
@@ -254,7 +284,7 @@ configs:
           }
       }
       loki.source.docker "logs_integrations_docker" {
-          host             = "unix:///var/run/docker.sock"
+          host             = "tcp://127.0.0.1:2375"
           targets          = discovery.docker.logs_integrations_docker.targets
           forward_to       = [loki.write.grafana_cloud_loki.receiver]
           relabel_rules    = discovery.relabel.logs_integrations_docker.rules