From 198329c008a624f4cbe517a0e92c69c72f16c698 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Sun, 4 Oct 2026 09:46:08 +0700 Subject: [PATCH] docs: move service issue notes from READMEs into docs/ Service READMEs now cover only what the service is and how to deploy it. Known issues, log noise and troubleshooting move to docs//, named after the service directory, so editing them never redeploys the service. Drop alloy's validate workflow, which never ran from a subdirectory. --- CLAUDE.md | 14 ++- README.md | 5 ++ alloy/.github/workflows/validate.yml | 88 ------------------- alloy/README.md | 11 --- .../duplicate-journal-and-syslog-lines.md | 18 ++++ .../known-noise-coolify-ssh-sessions.md | 2 +- .../alloy}/upstream-sources-of-truth.md | 8 +- docs/gitea-mirror/troubleshooting.md | 18 ++++ docs/opencode/known-noise-xdg-open.md | 11 +++ docs/paseo/troubleshooting.md | 9 ++ gitea-mirror/README.md | 10 --- opencode/README.md | 7 -- paseo/README.md | 3 +- 13 files changed, 78 insertions(+), 126 deletions(-) delete mode 100644 alloy/.github/workflows/validate.yml create mode 100644 docs/alloy/duplicate-journal-and-syslog-lines.md rename {alloy/docs => docs/alloy}/known-noise-coolify-ssh-sessions.md (99%) rename {alloy/docs => docs/alloy}/upstream-sources-of-truth.md (95%) create mode 100644 docs/gitea-mirror/troubleshooting.md create mode 100644 docs/opencode/known-noise-xdg-open.md create mode 100644 docs/paseo/troubleshooting.md diff --git a/CLAUDE.md b/CLAUDE.md index a5d2b9a..153dfbb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,9 +71,17 @@ containers use. - **Agent skills** for a service — maintenance scripts, runbooks — go in the root `.claude/skills//`, never in `/.claude/`. A skill there also only loads once a session touches that directory. -- **Anything else** that is not deploy input — CI workflows, research notes, - test fixtures — has no default home. Ask the user where it goes before - adding it to a service directory. +- **Docs** beyond the README go in the root `docs//`, named exactly + like the service directory; renaming a service renames its docs directory in + the same commit. The service `README.md` covers only what the service is and + how to deploy it — variables, storage, wiring, and the reasons behind its + configuration. Anything issue-related — known problems, log noise, + troubleshooting, investigations, upstream research and audits — goes in + `docs//`. The README does not link there; docs may link to the + service. +- **Anything else** that is not deploy input — CI workflows, test fixtures — + has no default home. Ask the user where it goes before adding it to a + service directory. ## Workspace services diff --git a/README.md b/README.md index 3eb42f9..406a830 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,11 @@ and are not repeated or linked from a service, so editing one service never touches another's directory — each is a separate Coolify app deploying on a `/**` watch path. +A service directory holds only what its deploy reads, so changing anything else +never redeploys it. Known issues, troubleshooting and research for a service +live in `docs//`, named after its directory; agent skills live in +`.claude/skills/`. + ## Upstream sources `sources/` holds checkouts of the upstream repositories behind these images, diff --git a/alloy/.github/workflows/validate.yml b/alloy/.github/workflows/validate.yml deleted file mode 100644 index adb4ea9..0000000 --- a/alloy/.github/workflows/validate.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: validate - -on: - push: - branches: [main] - pull_request: - -jobs: - validate: - runs-on: ubuntu-latest - env: - # Dummy values so `compose config` doesn't fail on the `:?required` guards. - ALLOY_HOSTNAME: ci - REMOTECFG_URL: https://example.com - REMOTECFG_ID: ci - REMOTECFG_USER: "1" - PROM_URL: https://example.com - PROM_USER: "1" - LOKI_URL: https://example.com - LOKI_USER: "1" - GRAFANA_TOKEN: x - steps: - - uses: actions/checkout@v4 - - - name: Validate compose YAML + env interpolation - run: docker compose config -q - - - name: Extract embedded Alloy config - uses: mikefarah/yq@v4 - with: - cmd: yq '.configs.alloy_config.content' compose.yml > config.alloy - - - name: Validate Alloy config syntax - run: | - docker run --rm -v "$PWD/config.alloy:/config.alloy:ro" \ - grafana/alloy:v1.16.1 fmt /config.alloy > /dev/null - - - name: Validate Alloy config arguments - run: | - docker run --rm \ - -e ALLOY_HOSTNAME -e REMOTECFG_URL -e REMOTECFG_ID -e REMOTECFG_USER \ - -e PROM_URL -e PROM_USER -e LOKI_URL -e LOKI_USER -e GRAFANA_TOKEN \ - -v "$PWD/config.alloy:/config.alloy:ro" \ - grafana/alloy:v1.16.1 validate /config.alloy - - - name: Validate Alloy config semantics (parse + load components) - # `fmt` and `validate` are static; this boots alloy with the same - # capability set, read-only rootfs and socket proxy the compose file - # uses, and reads the load log. Component runtime errors against dummy paths / - # endpoints are tolerated (level=error, level=warn) — only fail on - # config-level errors that prevent the graph from being built. - run: | - # /var/log/journal may not exist on the GHA runner; create empty - # dir so loki.source.journal can at least open it. - mkdir -p /tmp/empty-journal - docker volume create alloy-test-data >/dev/null - docker run -d --name alloy-test-proxy \ - --security-opt no-new-privileges:true \ - -e CONTAINERS=1 -e NETWORKS=1 -e IMAGES=1 -e INFO=1 -e VERSION=1 -e EVENTS=1 \ - -p 127.0.0.1:2375:2375 \ - -v /var/run/docker.sock:/var/run/docker.sock:ro \ - tecnativa/docker-socket-proxy:v0.5.0 - docker run -d --name alloy-test --network host \ - --cap-drop ALL --cap-add DAC_OVERRIDE \ - --security-opt no-new-privileges:true --read-only --tmpfs /tmp \ - -e ALLOY_HOSTNAME -e REMOTECFG_URL -e REMOTECFG_ID -e REMOTECFG_USER \ - -e PROM_URL -e PROM_USER -e LOKI_URL -e LOKI_USER -e GRAFANA_TOKEN \ - -v alloy-test-data:/var/lib/alloy/data \ - -v "$PWD/config.alloy:/etc/alloy/config.alloy:ro" \ - -v /proc:/rootproc:ro -v /sys:/sys:ro -v /:/rootfs:ro \ - -v /var/lib/docker:/var/lib/docker:ro \ - -v /tmp/empty-journal:/var/log/journal:ro \ - -v /etc/machine-id:/etc/machine-id:ro \ - grafana/alloy:v1.16.1 \ - run --server.http.listen-addr=127.0.0.1:12346 --disable-reporting \ - --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy - sleep 10 - docker logs alloy-test 2>&1 | tee /tmp/alloy.log - state=$(docker inspect -f '{{.State.Status}}' alloy-test 2>/dev/null || echo missing) - docker rm -f alloy-test alloy-test-proxy >/dev/null 2>&1 || true - docker volume rm alloy-test-data >/dev/null 2>&1 || true - if grep -Eq "could not (parse|load|build)|unknown component|component .* not registered|undefined reference|syntax error|expected .* found|cannot evaluate" /tmp/alloy.log; then - echo "::error::config-level error detected in alloy logs" - exit 1 - fi - if [ "$state" != "running" ]; then - echo "::warning::alloy exited (state=$state) but no config-error pattern matched — treating as runtime failure (tolerated)" - fi diff --git a/alloy/README.md b/alloy/README.md index 9fe29f7..3c4bb9c 100644 --- a/alloy/README.md +++ b/alloy/README.md @@ -31,11 +31,6 @@ integrations verbatim ([Linux Node](https://grafana.com/docs/grafana-cloud/monit [Docker](https://grafana.com/docs/grafana-cloud/monitor-infrastructure/integrations/integration-reference/integration-docker/#metrics)). Logs are unfiltered. -Where rsyslog mirrors journald into `/var/log/syslog` — the Debian and Ubuntu -default — the journal and file pipelines double-ship the same lines. Drop one -source on those hosts; the file-based one is the redundant one on systemd-only -stacks. - ## Environment All nine are required; `docker compose up` fails fast if any is unset. @@ -139,9 +134,3 @@ the disclosure one. `dockerproxy` mounts `/var/run/docker.sock:ro` and nothing else. -## Notes - -- [Upstream sources of truth](docs/upstream-sources-of-truth.md) — what this - follows, what is in scope, how to audit dashboard metric needs. -- [Coolify SSH session noise](docs/known-noise-coolify-ssh-sessions.md) — only - relevant on Coolify-managed hosts. diff --git a/docs/alloy/duplicate-journal-and-syslog-lines.md b/docs/alloy/duplicate-journal-and-syslog-lines.md new file mode 100644 index 0000000..e420e71 --- /dev/null +++ b/docs/alloy/duplicate-journal-and-syslog-lines.md @@ -0,0 +1,18 @@ +# Duplicate journal and syslog lines + +Applies to `alloy/compose.yml`. + +## Symptom + +The same log line arrives in Loki twice: once from `loki.source.journal`, once +from `loki.source.file`. + +## Cause + +Where rsyslog mirrors journald into `/var/log/syslog` — the Debian and Ubuntu +default — the journal and file pipelines both ship it. + +## Fix + +Drop one source on those hosts. On systemd-only stacks the file-based one is +the redundant one. diff --git a/alloy/docs/known-noise-coolify-ssh-sessions.md b/docs/alloy/known-noise-coolify-ssh-sessions.md similarity index 99% rename from alloy/docs/known-noise-coolify-ssh-sessions.md rename to docs/alloy/known-noise-coolify-ssh-sessions.md index d0dc5a4..fc37e6d 100644 --- a/alloy/docs/known-noise-coolify-ssh-sessions.md +++ b/docs/alloy/known-noise-coolify-ssh-sessions.md @@ -60,7 +60,7 @@ If the matching key's comment is `coolify` → this doc applies. ### Option A — drop the noise at Alloy (host stays as-is) -Edit the `loki.process "default"` block inside `journal_module` in `compose.yml`: +Edit the `loki.process "default"` block inside `journal_module` in `alloy/compose.yml`: ```alloy loki.process "default" { diff --git a/alloy/docs/upstream-sources-of-truth.md b/docs/alloy/upstream-sources-of-truth.md similarity index 95% rename from alloy/docs/upstream-sources-of-truth.md rename to docs/alloy/upstream-sources-of-truth.md index b616748..cb896b6 100644 --- a/alloy/docs/upstream-sources-of-truth.md +++ b/docs/alloy/upstream-sources-of-truth.md @@ -30,7 +30,7 @@ When the upstream integration changes (adds a metric, drops a panel, renames a l ## Upstream references -The two reference pages this repo's `compose.yml` mirrors: +The two reference pages `alloy/compose.yml` mirrors: - **Linux Node integration** — - **Docker integration** — @@ -42,7 +42,7 @@ Where official mixin dashboards exist they're tier-2 corroboration: ## Mapping our config to upstream -| Block in `compose.yml` | Upstream source | +| Block in `alloy/compose.yml` | Upstream source | |---|---| | `prometheus.exporter.unix` (collectors, mounts, fs/net excludes) | Linux Node integration page → "Configure Alloy" | | `prometheus.relabel "integrations_node_exporter"` (`keep` allowlist of 157 metrics) | Linux Node integration page → [Metrics](https://grafana.com/docs/grafana-cloud/monitor-infrastructure/integrations/integration-reference/integration-linux-node/#metrics) section, verbatim | @@ -56,11 +56,11 @@ Where official mixin dashboards exist they're tier-2 corroboration: 1. **Don't expand the metric set unilaterally.** If a panel in a Grafana Cloud dashboard requires a metric we don't ship, we'd add it — but the trigger is "the integration dashboard needs it, verified from a tier 1–4 source", not "node_exporter exposes it". 2. **Don't filter further than upstream does.** We ship at least what the upstream config does. Tightening (e.g. for cost) goes in a clearly-named overlay or a downstream env-specific config. -3. **Re-check on Alloy/integration major versions.** When bumping `grafana/alloy` image or when the Grafana Cloud integration revs, diff the upstream Alloy snippet against `compose.yml` and update. +3. **Re-check on Alloy/integration major versions.** When bumping `grafana/alloy` image or when the Grafana Cloud integration revs, diff the upstream Alloy snippet against `alloy/compose.yml` and update. ## Audit (2026-04-26) -Both keep-lists in `compose.yml` are copied verbatim from the **Metrics** section of each integration page (tier 1): +Both keep-lists in `alloy/compose.yml` are copied verbatim from the **Metrics** section of each integration page (tier 1): - **Linux-Node** — 157 raw metrics (`node_*`, `process_max_fds`, `process_open_fds`, `up`). The list also contains `instance:node_num_cpu:sum`, which is a recording-rule output computed server-side by Grafana Cloud's ruler — it's intentionally **not** in the keep-list because the agent doesn't produce it. - **Docker** — 16 metrics (`container_*`, `machine_memory_bytes`, `machine_scrape_error`, `up`). diff --git a/docs/gitea-mirror/troubleshooting.md b/docs/gitea-mirror/troubleshooting.md new file mode 100644 index 0000000..d9fa45b --- /dev/null +++ b/docs/gitea-mirror/troubleshooting.md @@ -0,0 +1,18 @@ +# gitea-mirror troubleshooting + +Applies to `gitea-mirror/compose.yml`. + +## Gitea cannot connect after changing `POSTGRES_PASSWORD` + +Postgres sets the password only when it first initialises `db-data`. Changing +`POSTGRES_PASSWORD` later breaks Gitea's connection until the role is altered +to match: + +```sh +docker compose exec db psql -U gitea -c "ALTER USER gitea PASSWORD '';" +``` + +## A redeploy kills clones in progress + +Gitea restarts and the clone dies with it. Avoid pushing to `gitea-mirror/` +while a large first mirror runs. diff --git a/docs/opencode/known-noise-xdg-open.md b/docs/opencode/known-noise-xdg-open.md new file mode 100644 index 0000000..87d5679 --- /dev/null +++ b/docs/opencode/known-noise-xdg-open.md @@ -0,0 +1,11 @@ +# Known noise: `xdg-open` stack trace on start + +Applies to `opencode/compose.yml`. + +The container logs a Bun stack trace ending in +`Executable not found in $PATH: "xdg-open"` on every start. `opencode web` +tries to open the UI in a local browser; there isn't one. It is noise — the +server is already listening by then, and the container keeps running. + +Installing `xdg-utils` would silence it at the cost of pulling X11 in and +tripling the image, which is not worth it for a log line. diff --git a/docs/paseo/troubleshooting.md b/docs/paseo/troubleshooting.md new file mode 100644 index 0000000..57aad0f --- /dev/null +++ b/docs/paseo/troubleshooting.md @@ -0,0 +1,9 @@ +# paseo troubleshooting + +Applies to `paseo/compose.yml`. + +## The pairing screen stays on `localhost:6767` + +After entering the address with its port, the UI may still show +`localhost:6767`: the old entry is cached in `localStorage`. Clear the site +data for the domain and enter it again. diff --git a/gitea-mirror/README.md b/gitea-mirror/README.md index d864a2f..d2e68c9 100644 --- a/gitea-mirror/README.md +++ b/gitea-mirror/README.md @@ -29,14 +29,6 @@ port, matching the two URL variables. | `ENCRYPTION_SECRET` | gitea-mirror | Encrypts the stored GitHub and Gitea tokens. Generate with `openssl rand -base64 48`. | | `GITEA_MIRROR_URL` | `BETTER_AUTH_URL`, `PUBLIC_BETTER_AUTH_URL`, `BETTER_AUTH_TRUSTED_ORIGINS` | Public URL of the mirror UI, no trailing slash. | -Postgres sets the password only when it first initialises `db-data`. Changing -`POSTGRES_PASSWORD` later breaks Gitea's connection until the role is altered -to match: - -```sh -docker compose exec db psql -U gitea -c "ALTER USER gitea PASSWORD '';" -``` - Behind a reverse proxy, gitea-mirror rejects sign-in with "invalid origin" unless all three Better Auth variables hold the external URL, so one variable feeds them all. @@ -62,8 +54,6 @@ install. record, which never syncs. `BUN_CONFIG_HTTP_IDLE_TIMEOUT` takes the same value as the clone timeout so the request outlives the clone. Bun caps it at 239 minutes, so a larger `GITEA_CLONE_TIMEOUT` stops helping there. -- **A redeploy kills clones in progress.** Gitea restarts and the clone dies - with it. Avoid pushing to this directory while a large first mirror runs. - **`gitea/gitea:28`.** Gitea publishes major tags; the major pin takes updates without a surprise major upgrade. - **`gitea-mirror:latest`** with `pull_policy: always`: upstream publishes no diff --git a/opencode/README.md b/opencode/README.md index 3a3d5b7..674cdeb 100644 --- a/opencode/README.md +++ b/opencode/README.md @@ -31,13 +31,6 @@ and survive a redeploy. Then open the domain and sign in with `OPENCODE_SERVER_USERNAME` and `OPENCODE_SERVER_PASSWORD`. -The container logs a Bun stack trace ending in -`Executable not found in $PATH: "xdg-open"` on every start. `opencode web` -tries to open the UI in a local browser; there isn't one. It is noise — the -server is already listening by then, and the container keeps running. -Installing `xdg-utils` would silence it at the cost of pulling X11 in and -tripling the image, which is not worth it for a log line. - ## Authentication `OPENCODE_SERVER_PASSWORD` is the only thing between the domain and a shell on diff --git a/paseo/README.md b/paseo/README.md index 23d156e..87f38da 100644 --- a/paseo/README.md +++ b/paseo/README.md @@ -25,8 +25,7 @@ The port must be typed by hand. The UI rejects a bare hostname, and the auto-connect hint does not help: the daemon builds it from the `Host` header, browsers drop the default `:443`, and the UI discards a hint with no port. What you see instead is its `localhost:6767` placeholder, which in a browser means -your own machine. If it stays on `localhost:6767` after you enter the address, -clear site data — the old entry is cached in `localStorage`. +your own machine. ## Environment