diff --git a/alloy/.github/workflows/validate.yml b/alloy/.github/workflows/validate.yml index 41ccfa4..4393841 100644 --- a/alloy/.github/workflows/validate.yml +++ b/alloy/.github/workflows/validate.yml @@ -35,31 +35,50 @@ jobs: docker run --rm -v "$PWD/config.alloy:/config.alloy:ro" \ grafana/alloy:v1.16.1 fmt /config.alloy > /dev/null + - name: Validate Alloy config arguments + run: | + docker run --rm \ + -e ALLOY_HOSTNAME -e REMOTECFG_URL -e REMOTECFG_ID -e REMOTECFG_USER \ + -e PROM_URL -e PROM_USER -e LOKI_URL -e LOKI_USER -e GRAFANA_TOKEN \ + -v "$PWD/config.alloy:/config.alloy:ro" \ + grafana/alloy:v1.16.1 validate /config.alloy + - name: Validate Alloy config semantics (parse + load components) - # `fmt` is syntax-only; this catches bad component refs, wrong arg - # types, and unknown component types by booting alloy and reading - # the load log. Component runtime errors against dummy paths / + # `fmt` and `validate` are static; this boots alloy with the same + # capability set, read-only rootfs and socket proxy the compose file + # uses, and reads the load log. Component runtime errors against dummy paths / # endpoints are tolerated (level=error, level=warn) — only fail on # config-level errors that prevent the graph from being built. run: | # /var/log/journal may not exist on the GHA runner; create empty # dir so loki.source.journal can at least open it. mkdir -p /tmp/empty-journal - docker run -d --name alloy-test --privileged \ + docker volume create alloy-test-data >/dev/null + docker run -d --name alloy-test-proxy \ + --security-opt no-new-privileges:true \ + -e CONTAINERS=1 -e NETWORKS=1 -e IMAGES=1 -e INFO=1 -e VERSION=1 -e EVENTS=1 \ + -p 127.0.0.1:2375:2375 \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + tecnativa/docker-socket-proxy:v0.5.0 + docker run -d --name alloy-test --network host \ + --cap-drop ALL --cap-add DAC_OVERRIDE \ + --security-opt no-new-privileges:true --read-only --tmpfs /tmp \ -e ALLOY_HOSTNAME -e REMOTECFG_URL -e REMOTECFG_ID -e REMOTECFG_USER \ -e PROM_URL -e PROM_USER -e LOKI_URL -e LOKI_USER -e GRAFANA_TOKEN \ + -v alloy-test-data:/var/lib/alloy/data \ -v "$PWD/config.alloy:/etc/alloy/config.alloy:ro" \ -v /proc:/rootproc:ro -v /sys:/sys:ro -v /:/rootfs:ro \ - -v /var/run/docker.sock:/var/run/docker.sock \ -v /var/lib/docker:/var/lib/docker:ro \ -v /tmp/empty-journal:/var/log/journal:ro \ -v /etc/machine-id:/etc/machine-id:ro \ grafana/alloy:v1.16.1 \ - run --server.http.listen-addr=127.0.0.1:12345 --disable-reporting /etc/alloy/config.alloy + run --server.http.listen-addr=127.0.0.1:12346 --disable-reporting \ + --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy sleep 10 docker logs alloy-test 2>&1 | tee /tmp/alloy.log state=$(docker inspect -f '{{.State.Status}}' alloy-test 2>/dev/null || echo missing) - docker rm -f alloy-test >/dev/null 2>&1 || true + docker rm -f alloy-test alloy-test-proxy >/dev/null 2>&1 || true + docker volume rm alloy-test-data >/dev/null 2>&1 || true if grep -Eq "could not (parse|load|build)|unknown component|component .* not registered|undefined reference|syntax error|expected .* found|cannot evaluate" /tmp/alloy.log; then echo "::error::config-level error detected in alloy logs" exit 1 diff --git a/alloy/README.md b/alloy/README.md index 2ef47ed..720f9dc 100644 --- a/alloy/README.md +++ b/alloy/README.md @@ -5,7 +5,8 @@ container telemetry to Grafana Cloud, with remote config from Grafana Fleet Management. One container runs both the `node_exporter` (host) and `cadvisor` (container) -collectors. The Alloy config is embedded inline via Compose `configs:`, so +collectors. A second, tiny container proxies a read-only slice of the Docker +API to it. The Alloy config is embedded inline via Compose `configs:`, so there is no `config.alloy` on disk, and every setting comes from a shell variable rather than a `.env` file. @@ -62,11 +63,59 @@ per host. Filter in Grafana with `instance=~"..."`. ## Privileges -Runs `privileged: true` with `network_mode: host`, matching the upstream -Grafana Cloud docker integration. Host networking is what lets -`prometheus.exporter.unix` report the host's real interfaces (`eth0`…) instead -of the container's veth pair. To tighten this, see the upstream Alloy docker -integration docs. +The upstream Grafana Cloud docker integration runs `privileged: true` with the +raw Docker socket mounted read-write. Nothing in this config needs that, and +the combination is host-root-equivalent: `privileged` grants every capability +and unmasks `/proc` and `/sys`, and a container that can talk to the Docker +socket can start another container that mounts `/` writable. The socket's `:ro` +flag does not help — it stops the socket *file* being replaced, not the API +being used. + +What it runs instead: + +| Setting | Reason | +| --- | --- | +| `cap_drop: [ALL]` + `cap_add: [DAC_OVERRIDE]` | The image's entrypoint runs as uid 0 and reads host files owned by other users — the journal, paths under `/rootfs`, `/var/log`. Dropping every capability leaves it unable to open them, and unable to create its own storage directory. `DAC_OVERRIDE` restores exactly that and nothing else; `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `MKNOD` and the rest stay dropped. | +| `no-new-privileges:true` | No setuid binary in the image can regain what was dropped. | +| `read_only: true` with `tmpfs: /tmp` | Only the `alloy-data` volume is writable. | +| `mem_limit: 2g`, `pids_limit: 512` | Steady state is around 900 MB; the limit stops a leak taking the host down with it. | +| `dockerproxy` instead of `/var/run/docker.sock` | See below. | + +`network_mode: host` stays. `/proc/net` is a symlink to `/proc/self/net` and +resolves against the reading process's network namespace, so bind-mounting the +host's `/proc` to `/rootproc` is not enough — without host networking the +netdev, netstat, sockstat and conntrack families would describe the container's +veth pair instead of `eth0`. That is roughly 45 of the 157 kept metrics. + +`/:/rootfs:ro` also stays, and is the widest remaining exposure: the container +can read every file on the host. The filesystem collector needs to `statfs()` +each mount point, and a bind mount cannot grant that without granting reads. +Dropping it would cost disk-space monitoring. Treat the container as +secret-bearing — it holds `GRAFANA_TOKEN` regardless. + +## Docker API access + +`prometheus.exporter.cadvisor` and `loki.source.docker` both need the Docker +API, so it cannot simply be removed. `dockerproxy` runs +[tecnativa/docker-socket-proxy](https://github.com/Tecnativa/docker-socket-proxy) +with the socket mounted read-only and exposes it on `127.0.0.1:2375`, which +Alloy reaches over host networking. + +`POST` is revoked by default in that image, which is the point: container +create, `exec`, start and kill are all refused with 403, so a compromise of +Alloy can no longer become root on the host. The granted sections are the ones +the two components actually call: + +| Variable | Called by | +| --- | --- | +| `CONTAINERS` | `discovery.docker` listing, `loki.source.docker` inspect and log read, cadvisor metadata | +| `NETWORKS` | `discovery.docker` — Prometheus' Docker SD resolves network names per container, and returns **zero targets** without it | +| `IMAGES`, `INFO`, `VERSION` | cadvisor | +| `EVENTS` | cadvisor container watch (granted by default) | + +What this does not do: `GET /containers/{id}/json` still returns every +container's environment variables. The proxy closes the escalation path, not +the disclosure one. ## Mounts @@ -76,11 +125,12 @@ integration docs. | `/sys:/sys:ro` | node-exporter and cadvisor cgroups | | `/:/rootfs:ro` | filesystem collector, via `rootfs_path` | | `/dev/disk/:/dev/disk:ro` | node-exporter diskstats device labels | -| `/var/run/docker.sock` | `discovery.docker` and `loki.source.docker` | | `/var/lib/docker:ro` | cadvisor container metadata | | `/var/log:/var/log:ro` | `loki.source.journal` and `loki.source.file` | | `/etc/machine-id:ro` | Stable host id for the journal reader | -| `alloy-data` | WAL and remotecfg cache | +| `alloy-data` | WAL and remotecfg cache — the only writable path | + +`dockerproxy` mounts `/var/run/docker.sock:ro` and nothing else. ## Notes diff --git a/alloy/docker-compose.yml b/alloy/docker-compose.yml index 25b4f55..65da101 100644 --- a/alloy/docker-compose.yml +++ b/alloy/docker-compose.yml @@ -8,8 +8,18 @@ services: container_name: alloy restart: unless-stopped hostname: ${ALLOY_HOSTNAME:?required} - privileged: true network_mode: host # node_exporter netdev/netstat sees real host interfaces (eth0…) not veth + cap_drop: [ALL] + cap_add: [DAC_OVERRIDE] # read host files owned by other uids: journal, /rootfs, /var/log + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp + mem_limit: 2g + pids_limit: 512 + depends_on: + - dockerproxy environment: ALLOY_DEPLOY_MODE: docker REMOTECFG_URL: ${REMOTECFG_URL:?required} @@ -23,7 +33,6 @@ services: volumes: - alloy-data:/var/lib/alloy/data - /proc:/rootproc:ro - - /var/run/docker.sock:/var/run/docker.sock - /sys:/sys:ro - /:/rootfs:ro - /dev/disk/:/dev/disk:ro @@ -34,6 +43,26 @@ services: - { source: alloy_config, target: /etc/alloy/config.alloy } command: 'run --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy' + dockerproxy: + image: tecnativa/docker-socket-proxy:v0.5.0 + container_name: alloy-dockerproxy + restart: unless-stopped + security_opt: + - no-new-privileges:true + mem_limit: 64m + pids_limit: 64 + environment: # the API sections cadvisor and the log tailer read; POST stays revoked + CONTAINERS: 1 + NETWORKS: 1 + IMAGES: 1 + INFO: 1 + VERSION: 1 + EVENTS: 1 + ports: + - 127.0.0.1:2375:2375 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + volumes: alloy-data: @@ -195,6 +224,7 @@ configs: prometheus.exporter.cadvisor "integrations_cadvisor" { docker_only = true + docker_host = "tcp://127.0.0.1:2375" } discovery.relabel "integrations_cadvisor" { targets = prometheus.exporter.cadvisor.integrations_cadvisor.targets @@ -226,7 +256,7 @@ configs: } discovery.docker "logs_integrations_docker" { - host = "unix:///var/run/docker.sock" + host = "tcp://127.0.0.1:2375" refresh_interval = "5s" } discovery.relabel "logs_integrations_docker" { @@ -254,7 +284,7 @@ configs: } } loki.source.docker "logs_integrations_docker" { - host = "unix:///var/run/docker.sock" + host = "tcp://127.0.0.1:2375" targets = discovery.docker.logs_integrations_docker.targets forward_to = [loki.write.grafana_cloud_loki.receiver] relabel_rules = discovery.relabel.logs_integrations_docker.rules