From 0ef059dc31e3f1b1d4b60e524466fe0f7622f74f Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 18 Sep 2026 17:28:08 +0700 Subject: [PATCH] refactor(alloy): drop privileged and proxy the docker socket read-only Replace privileged: true with cap_drop ALL plus DAC_OVERRIDE, no-new-privileges, a read-only rootfs and memory/pid limits. DAC_OVERRIDE is what lets the uid-0 entrypoint create its storage directory and read the journal and /rootfs; every other capability stays dropped. Route prometheus.exporter.cadvisor, discovery.docker and loki.source.docker through a docker-socket-proxy sidecar on 127.0.0.1:2375 instead of bind-mounting the socket. POST is refused there, so container create and exec are no longer reachable. NETWORKS is granted because Docker SD resolves network names per container and returns no targets without it. Add an alloy validate step to CI and boot the test container with the shipped capability set, read-only rootfs and proxy rather than --privileged. --- alloy/.github/workflows/validate.yml | 33 +++++++++++--- alloy/README.md | 66 ++++++++++++++++++++++++---- alloy/docker-compose.yml | 38 ++++++++++++++-- 3 files changed, 118 insertions(+), 19 deletions(-) diff --git a/alloy/.github/workflows/validate.yml b/alloy/.github/workflows/validate.yml index 41ccfa4..4393841 100644 --- a/alloy/.github/workflows/validate.yml +++ b/alloy/.github/workflows/validate.yml @@ -35,31 +35,50 @@ jobs: docker run --rm -v "$PWD/config.alloy:/config.alloy:ro" \ grafana/alloy:v1.16.1 fmt /config.alloy > /dev/null + - name: Validate Alloy config arguments + run: | + docker run --rm \ + -e ALLOY_HOSTNAME -e REMOTECFG_URL -e REMOTECFG_ID -e REMOTECFG_USER \ + -e PROM_URL -e PROM_USER -e LOKI_URL -e LOKI_USER -e GRAFANA_TOKEN \ + -v "$PWD/config.alloy:/config.alloy:ro" \ + grafana/alloy:v1.16.1 validate /config.alloy + - name: Validate Alloy config semantics (parse + load components) - # `fmt` is syntax-only; this catches bad component refs, wrong arg - # types, and unknown component types by booting alloy and reading - # the load log. Component runtime errors against dummy paths / + # `fmt` and `validate` are static; this boots alloy with the same + # capability set, read-only rootfs and socket proxy the compose file + # uses, and reads the load log. Component runtime errors against dummy paths / # endpoints are tolerated (level=error, level=warn) — only fail on # config-level errors that prevent the graph from being built. run: | # /var/log/journal may not exist on the GHA runner; create empty # dir so loki.source.journal can at least open it. mkdir -p /tmp/empty-journal - docker run -d --name alloy-test --privileged \ + docker volume create alloy-test-data >/dev/null + docker run -d --name alloy-test-proxy \ + --security-opt no-new-privileges:true \ + -e CONTAINERS=1 -e NETWORKS=1 -e IMAGES=1 -e INFO=1 -e VERSION=1 -e EVENTS=1 \ + -p 127.0.0.1:2375:2375 \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + tecnativa/docker-socket-proxy:v0.5.0 + docker run -d --name alloy-test --network host \ + --cap-drop ALL --cap-add DAC_OVERRIDE \ + --security-opt no-new-privileges:true --read-only --tmpfs /tmp \ -e ALLOY_HOSTNAME -e REMOTECFG_URL -e REMOTECFG_ID -e REMOTECFG_USER \ -e PROM_URL -e PROM_USER -e LOKI_URL -e LOKI_USER -e GRAFANA_TOKEN \ + -v alloy-test-data:/var/lib/alloy/data \ -v "$PWD/config.alloy:/etc/alloy/config.alloy:ro" \ -v /proc:/rootproc:ro -v /sys:/sys:ro -v /:/rootfs:ro \ - -v /var/run/docker.sock:/var/run/docker.sock \ -v /var/lib/docker:/var/lib/docker:ro \ -v /tmp/empty-journal:/var/log/journal:ro \ -v /etc/machine-id:/etc/machine-id:ro \ grafana/alloy:v1.16.1 \ - run --server.http.listen-addr=127.0.0.1:12345 --disable-reporting /etc/alloy/config.alloy + run --server.http.listen-addr=127.0.0.1:12346 --disable-reporting \ + --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy sleep 10 docker logs alloy-test 2>&1 | tee /tmp/alloy.log state=$(docker inspect -f '{{.State.Status}}' alloy-test 2>/dev/null || echo missing) - docker rm -f alloy-test >/dev/null 2>&1 || true + docker rm -f alloy-test alloy-test-proxy >/dev/null 2>&1 || true + docker volume rm alloy-test-data >/dev/null 2>&1 || true if grep -Eq "could not (parse|load|build)|unknown component|component .* not registered|undefined reference|syntax error|expected .* found|cannot evaluate" /tmp/alloy.log; then echo "::error::config-level error detected in alloy logs" exit 1 diff --git a/alloy/README.md b/alloy/README.md index 2ef47ed..720f9dc 100644 --- a/alloy/README.md +++ b/alloy/README.md @@ -5,7 +5,8 @@ container telemetry to Grafana Cloud, with remote config from Grafana Fleet Management. One container runs both the `node_exporter` (host) and `cadvisor` (container) -collectors. The Alloy config is embedded inline via Compose `configs:`, so +collectors. A second, tiny container proxies a read-only slice of the Docker +API to it. The Alloy config is embedded inline via Compose `configs:`, so there is no `config.alloy` on disk, and every setting comes from a shell variable rather than a `.env` file. @@ -62,11 +63,59 @@ per host. Filter in Grafana with `instance=~"..."`. ## Privileges -Runs `privileged: true` with `network_mode: host`, matching the upstream -Grafana Cloud docker integration. Host networking is what lets -`prometheus.exporter.unix` report the host's real interfaces (`eth0`…) instead -of the container's veth pair. To tighten this, see the upstream Alloy docker -integration docs. +The upstream Grafana Cloud docker integration runs `privileged: true` with the +raw Docker socket mounted read-write. Nothing in this config needs that, and +the combination is host-root-equivalent: `privileged` grants every capability +and unmasks `/proc` and `/sys`, and a container that can talk to the Docker +socket can start another container that mounts `/` writable. The socket's `:ro` +flag does not help — it stops the socket *file* being replaced, not the API +being used. + +What it runs instead: + +| Setting | Reason | +| --- | --- | +| `cap_drop: [ALL]` + `cap_add: [DAC_OVERRIDE]` | The image's entrypoint runs as uid 0 and reads host files owned by other users — the journal, paths under `/rootfs`, `/var/log`. Dropping every capability leaves it unable to open them, and unable to create its own storage directory. `DAC_OVERRIDE` restores exactly that and nothing else; `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `MKNOD` and the rest stay dropped. | +| `no-new-privileges:true` | No setuid binary in the image can regain what was dropped. | +| `read_only: true` with `tmpfs: /tmp` | Only the `alloy-data` volume is writable. | +| `mem_limit: 2g`, `pids_limit: 512` | Steady state is around 900 MB; the limit stops a leak taking the host down with it. | +| `dockerproxy` instead of `/var/run/docker.sock` | See below. | + +`network_mode: host` stays. `/proc/net` is a symlink to `/proc/self/net` and +resolves against the reading process's network namespace, so bind-mounting the +host's `/proc` to `/rootproc` is not enough — without host networking the +netdev, netstat, sockstat and conntrack families would describe the container's +veth pair instead of `eth0`. That is roughly 45 of the 157 kept metrics. + +`/:/rootfs:ro` also stays, and is the widest remaining exposure: the container +can read every file on the host. The filesystem collector needs to `statfs()` +each mount point, and a bind mount cannot grant that without granting reads. +Dropping it would cost disk-space monitoring. Treat the container as +secret-bearing — it holds `GRAFANA_TOKEN` regardless. + +## Docker API access + +`prometheus.exporter.cadvisor` and `loki.source.docker` both need the Docker +API, so it cannot simply be removed. `dockerproxy` runs +[tecnativa/docker-socket-proxy](https://github.com/Tecnativa/docker-socket-proxy) +with the socket mounted read-only and exposes it on `127.0.0.1:2375`, which +Alloy reaches over host networking. + +`POST` is revoked by default in that image, which is the point: container +create, `exec`, start and kill are all refused with 403, so a compromise of +Alloy can no longer become root on the host. The granted sections are the ones +the two components actually call: + +| Variable | Called by | +| --- | --- | +| `CONTAINERS` | `discovery.docker` listing, `loki.source.docker` inspect and log read, cadvisor metadata | +| `NETWORKS` | `discovery.docker` — Prometheus' Docker SD resolves network names per container, and returns **zero targets** without it | +| `IMAGES`, `INFO`, `VERSION` | cadvisor | +| `EVENTS` | cadvisor container watch (granted by default) | + +What this does not do: `GET /containers/{id}/json` still returns every +container's environment variables. The proxy closes the escalation path, not +the disclosure one. ## Mounts @@ -76,11 +125,12 @@ integration docs. | `/sys:/sys:ro` | node-exporter and cadvisor cgroups | | `/:/rootfs:ro` | filesystem collector, via `rootfs_path` | | `/dev/disk/:/dev/disk:ro` | node-exporter diskstats device labels | -| `/var/run/docker.sock` | `discovery.docker` and `loki.source.docker` | | `/var/lib/docker:ro` | cadvisor container metadata | | `/var/log:/var/log:ro` | `loki.source.journal` and `loki.source.file` | | `/etc/machine-id:ro` | Stable host id for the journal reader | -| `alloy-data` | WAL and remotecfg cache | +| `alloy-data` | WAL and remotecfg cache — the only writable path | + +`dockerproxy` mounts `/var/run/docker.sock:ro` and nothing else. ## Notes diff --git a/alloy/docker-compose.yml b/alloy/docker-compose.yml index 25b4f55..65da101 100644 --- a/alloy/docker-compose.yml +++ b/alloy/docker-compose.yml @@ -8,8 +8,18 @@ services: container_name: alloy restart: unless-stopped hostname: ${ALLOY_HOSTNAME:?required} - privileged: true network_mode: host # node_exporter netdev/netstat sees real host interfaces (eth0…) not veth + cap_drop: [ALL] + cap_add: [DAC_OVERRIDE] # read host files owned by other uids: journal, /rootfs, /var/log + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp + mem_limit: 2g + pids_limit: 512 + depends_on: + - dockerproxy environment: ALLOY_DEPLOY_MODE: docker REMOTECFG_URL: ${REMOTECFG_URL:?required} @@ -23,7 +33,6 @@ services: volumes: - alloy-data:/var/lib/alloy/data - /proc:/rootproc:ro - - /var/run/docker.sock:/var/run/docker.sock - /sys:/sys:ro - /:/rootfs:ro - /dev/disk/:/dev/disk:ro @@ -34,6 +43,26 @@ services: - { source: alloy_config, target: /etc/alloy/config.alloy } command: 'run --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy' + dockerproxy: + image: tecnativa/docker-socket-proxy:v0.5.0 + container_name: alloy-dockerproxy + restart: unless-stopped + security_opt: + - no-new-privileges:true + mem_limit: 64m + pids_limit: 64 + environment: # the API sections cadvisor and the log tailer read; POST stays revoked + CONTAINERS: 1 + NETWORKS: 1 + IMAGES: 1 + INFO: 1 + VERSION: 1 + EVENTS: 1 + ports: + - 127.0.0.1:2375:2375 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + volumes: alloy-data: @@ -195,6 +224,7 @@ configs: prometheus.exporter.cadvisor "integrations_cadvisor" { docker_only = true + docker_host = "tcp://127.0.0.1:2375" } discovery.relabel "integrations_cadvisor" { targets = prometheus.exporter.cadvisor.integrations_cadvisor.targets @@ -226,7 +256,7 @@ configs: } discovery.docker "logs_integrations_docker" { - host = "unix:///var/run/docker.sock" + host = "tcp://127.0.0.1:2375" refresh_interval = "5s" } discovery.relabel "logs_integrations_docker" { @@ -254,7 +284,7 @@ configs: } } loki.source.docker "logs_integrations_docker" { - host = "unix:///var/run/docker.sock" + host = "tcp://127.0.0.1:2375" targets = discovery.docker.logs_integrations_docker.targets forward_to = [loki.write.grafana_cloud_loki.receiver] relabel_rules = discovery.relabel.logs_integrations_docker.rules