composes/alloy/docker-compose.yml

# Required shell env vars (export before `docker compose up`):
#   ALLOY_HOSTNAME  PROM_URL  PROM_USER  LOKI_URL  LOKI_USER  GRAFANA_TOKEN
# UI reachable only from inside the Docker network: http://alloy:12345

services:
  alloy:
    image: grafana/alloy:v1.10.0
    container_name: alloy
    restart: unless-stopped
    hostname: ${ALLOY_HOSTNAME:?required}
    user: "0:0"                                   # root needed for docker.sock + journal files
    environment:
      PROM_URL:      ${PROM_URL:?required}
      PROM_USER:     ${PROM_USER:?required}
      LOKI_URL:      ${LOKI_URL:?required}
      LOKI_USER:     ${LOKI_USER:?required}
      GRAFANA_TOKEN: ${GRAFANA_TOKEN:?required}
    security_opt: [ "no-new-privileges:true" ]
    cap_drop:     [ ALL ]
    cap_add:      [ DAC_OVERRIDE, SYSLOG ]        # DAC_OVERRIDE: write alloy-owned /var/lib/alloy/data + read root-owned mounts. SYSLOG: read /dev/kmsg when kernel.dmesg_restrict=1.
    volumes:
      - alloy-data:/var/lib/alloy/data                              # remotecfg + storage state
      - /var/run/docker.sock:/var/run/docker.sock:ro                # discovery.docker + loki.source.docker (streams log API)
      - /var/lib/docker:/var/lib/docker:ro                          # cadvisor container metadata
      - /proc:/host/proc:ro                                         # node-exporter cpu/mem/load/net
      - /sys:/host/sys:ro                                           # node-exporter + cadvisor cgroups
      - /:/rootfs:ro,rslave                                         # filesystem collector (rslave picks up new mounts)
      - /var/log/journal:/var/log/journal:ro                        # loki.source.journal
      - /etc/machine-id:/etc/machine-id:ro                          # stable host id for journal reader
      - /dev/kmsg:/dev/kmsg:ro                                      # cadvisor OOM event detection
      - /run/udev/data:/run/udev/data:ro                            # node-exporter diskstats device labels (model/serial/WWN)
    configs:
      - { source: alloy_config, target: /etc/alloy/config.alloy }
    command: [ run, --server.http.listen-addr=0.0.0.0:12345, --storage.path=/var/lib/alloy/data, /etc/alloy/config.alloy ]

volumes:
  alloy-data:

configs:
  alloy_config:
    content: |
      prometheus.remote_write "gc" {
        endpoint {
          url = sys.env("PROM_URL")
          basic_auth {
            username = sys.env("PROM_USER")
            password = sys.env("GRAFANA_TOKEN")
          }
        }
      }

      loki.write "gc" {
        external_labels = { instance = constants.hostname }
        endpoint {
          url        = sys.env("LOKI_URL")
          batch_wait = "5s"
          batch_size = "1MiB"
          basic_auth {
            username = sys.env("LOKI_USER")
            password = sys.env("GRAFANA_TOKEN")
          }
        }
      }

      prometheus.exporter.unix "host" {
        rootfs_path = "/rootfs"
        procfs_path = "/host/proc"
        sysfs_path  = "/host/sys"
        filesystem  { mount_points_exclude = "^/(dev|proc|sys|run|var/lib/docker)($|/)" }
        netdev      { device_exclude       = "^(veth.*|docker.*|br-.*|lo)$" }
      }

      prometheus.exporter.cadvisor "containers" {
        docker_only = true
      }

      prometheus.scrape "metrics" {
        targets = array.concat(
          prometheus.exporter.unix.host.targets,
          prometheus.exporter.cadvisor.containers.targets,
        )
        forward_to      = [prometheus.relabel.filter.receiver]
        scrape_interval = "60s"
      }

      // Sets instance label + keeps only metrics used by standard dashboards.
      // keep-filter essential for the 10k active-series cap on Grafana Cloud Free.
      prometheus.relabel "filter" {
        forward_to = [prometheus.remote_write.gc.receiver]
        rule {
          target_label = "instance"
          replacement  = constants.hostname
        }
        rule {
          source_labels = ["__name__"]
          action        = "keep"
          regex         = "up|node_(cpu_seconds_total|load(1|5|15)|boot_time_seconds|uname_info|memory_(MemTotal|MemFree|MemAvailable|Cached|Buffers|SwapTotal|SwapFree)_bytes|filesystem_(size|avail)_bytes|filesystem_(files|files_free|readonly)|disk_(reads|writes)_completed_total|disk_(read|written)_bytes_total|disk_io_time(_weighted)?_seconds_total|network_(receive|transmit)_(bytes|errs|drop)_total)|container_(cpu_usage_seconds_total|memory_(usage|working_set)_bytes|fs_(usage|limit)_bytes|network_(receive|transmit)_bytes_total|last_seen)|machine_(memory_bytes|scrape_error)"
        }
      }

      discovery.docker "containers" {
        host             = "unix:///var/run/docker.sock"
        refresh_interval = "5s"
      }

      discovery.relabel "container_logs" {
        targets = discovery.docker.containers.targets
        rule {
          source_labels = ["__meta_docker_container_name"]
          regex         = "/(.*)"
          target_label  = "container"
        }
        rule {
          source_labels = ["__meta_docker_container_log_stream"]
          target_label  = "stream"
        }
      }

      // Grafana Cloud Loki rejects entries older than 7 days (HTTP 400).
      // Long-running containers replay weeks of backlog on first start — drop
      // anything older than 6 days to stay safely under the cutoff.
      loki.process "drop_old" {
        forward_to = [loki.write.gc.receiver]
        stage.drop {
          older_than          = "144h"
          drop_counter_reason = "entry_too_old"
        }
      }

      loki.source.docker "containers" {
        host       = "unix:///var/run/docker.sock"
        targets    = discovery.relabel.container_logs.output
        forward_to = [loki.process.drop_old.receiver]
      }

      loki.relabel "journal" {
        forward_to = []
        rule {
          source_labels = ["__journal__systemd_unit"]
          target_label  = "unit"
        }
        rule {
          source_labels = ["__journal_priority_keyword"]
          target_label  = "level"
        }
      }

      loki.source.journal "system" {
        max_age       = "12h"
        path          = "/var/log/journal"
        forward_to    = [loki.write.gc.receiver]
        relabel_rules = loki.relabel.journal.rules
        labels        = { job = "systemd-journal" }
      }