tiennm99/goclaw
1
0 Fork 0
mirror of https://github.com/tiennm99/goclaw.git synced 2026-10-16 00:14:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #33 from digitopvn/dev

Browse Source 
release: promote v3.12.0 official
This commit is contained in:
Duy /zuey/ authored and GitHub committed 2026-05-20 17:09:53 +07:00
2 parents 364d2d346f 532ff91d8e
commit 0c7add6736
455 files changed
+45961 -1751
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.env.example
+3
View File
@@ -5,6 +5,9 @@
# LLM provider API keys: configure via the web dashboard setup wizard.
# --- Gateway (required) ---
# Required for Docker/external binds. Run ./prepare-env.sh to generate.
# Local loopback-only development may opt into empty-token mode with:
# GOCLAW_ALLOW_INSECURE_NO_AUTH=1
GOCLAW_GATEWAY_TOKEN=
GOCLAW_ENCRYPTION_KEY=
POSTGRES_PASSWORD=
.github/workflows/dev-beta-release.yaml
+327
View File
@@ -0,0 +1,327 @@
name: Dev CI and Beta Release
on:
push:
branches: [dev]
workflow_dispatch:
permissions:
contents: read
concurrency:
group: dev-beta-release-${{ github.ref }}
cancel-in-progress: false
env:
GHCR_IMAGE: ghcr.io/${{ github.repository }}
DOCKERHUB_IMAGE: digitop/goclaw
INITIAL_VERSION: 3.11.3
PRERELEASE_ID: beta
jobs:
go:
runs-on: ubuntu-latest
services:
pg:
image: pgvector/pgvector:pg18
env:
POSTGRES_PASSWORD: test
POSTGRES_DB: goclaw_test
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 3s
--health-retries 10
env:
TEST_DATABASE_URL: postgres://postgres:test@localhost:5432/goclaw_test?sslmode=disable
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache-dependency-path: go.sum
- run: go build ./...
- run: go build -tags sqliteonly ./...
- run: go vet ./...
- name: Unit tests
run: go test -race -timeout=5m -coverpkg=./... -coverprofile=coverage.out ./...
- name: Invariant tests (P0)
run: go test -race -timeout=90s -tags integration ./tests/invariants/...
- name: Contract tests (P1)
run: go test -race -timeout=90s -tags integration ./tests/contracts/... || echo "::warning::Contract tests skipped (no server configured)"
continue-on-error: true
- name: Integration tests
run: go test -race -timeout=180s -tags integration ./tests/integration/
- name: Coverage summary
run: go tool cover -func=coverage.out | tail -1
web:
runs-on: ubuntu-latest
defaults:
run:
working-directory: ui/web
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
with:
version: 10
- uses: actions/setup-node@v4
with:
node-version: 22
cache: pnpm
cache-dependency-path: ui/web/pnpm-lock.yaml
- run: pnpm install --frozen-lockfile
- run: pnpm lint
- run: pnpm build
beta_version:
needs: [go, web]
if: github.ref == 'refs/heads/dev'
runs-on: ubuntu-latest
permissions:
contents: write
outputs:
released: ${{ steps.version.outputs.released }}
version: ${{ steps.version.outputs.version }}
tag: ${{ steps.version.outputs.tag }}
notes_path: ${{ steps.version.outputs.notes_path }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Fetch upstream release tags
run: git fetch --force --tags https://github.com/nextlevelbuilder/goclaw.git "refs/tags/v*:refs/tags/v*"
- name: Compute semantic beta version
id: version
run: node scripts/ci/semantic-beta-version.mjs
- name: Create or verify beta tag
if: steps.version.outputs.released == 'true'
env:
TAG: ${{ steps.version.outputs.tag }}
run: |
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
if ! git rev-parse "$TAG" >/dev/null 2>&1; then
git tag -a "$TAG" -m "Release $TAG"
git push origin "$TAG"
fi
- name: Upload release notes
if: steps.version.outputs.released == 'true'
uses: actions/upload-artifact@v4
with:
name: release-notes
path: ${{ steps.version.outputs.notes_path }}
build_binaries:
needs: beta_version
if: needs.beta_version.outputs.released == 'true'
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- goos: linux
goarch: amd64
- goos: linux
goarch: arm64
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.beta_version.outputs.tag }}
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache-dependency-path: go.sum
- uses: actions/setup-node@v4
with:
node-version: 22
- name: Build web UI
run: |
corepack enable && corepack prepare pnpm@10.28.2 --activate
cd ui/web && pnpm install --frozen-lockfile && pnpm build && cd ../..
mkdir -p internal/webui/dist
cp -r ui/web/dist/* internal/webui/dist/
- name: Build binary
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
VERSION: ${{ needs.beta_version.outputs.tag }}
run: |
CGO_ENABLED=0 go build -tags embedui \
-ldflags="-s -w -X github.com/nextlevelbuilder/goclaw/cmd.Version=${VERSION}" \
-o goclaw .
tar -czf "goclaw-${VERSION}-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz" goclaw migrations/
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: binary-${{ matrix.goos }}-${{ matrix.goarch }}
path: goclaw-*.tar.gz
publish_release:
needs: [beta_version, build_binaries, promote_beta_aliases]
if: needs.beta_version.outputs.released == 'true'
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
pattern: binary-*
path: artifacts
merge-multiple: true
- name: Download release notes
uses: actions/download-artifact@v4
with:
name: release-notes
path: release-notes
- name: Publish prerelease
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
TAG: ${{ needs.beta_version.outputs.tag }}
run: |
if gh release view "$TAG" >/dev/null 2>&1; then
gh release edit "$TAG" \
--title "GoClaw $TAG" \
--notes-file release-notes/release-notes.md \
--prerelease
else
gh release create "$TAG" \
--title "GoClaw $TAG" \
--notes-file release-notes/release-notes.md \
--prerelease
fi
gh release upload "$TAG" artifacts/* --clobber
docker_images:
needs: beta_version
if: needs.beta_version.outputs.released == 'true'
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
strategy:
fail-fast: false
matrix:
include:
- variant: latest
suffix: ""
enable_otel: "false"
enable_embedui: "true"
enable_python: "true"
enable_full_skills: "false"
- variant: full
suffix: "-full"
enable_otel: "false"
enable_embedui: "true"
enable_python: "true"
enable_full_skills: "true"
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.beta_version.outputs.tag }}
- uses: docker/setup-qemu-action@v3
- uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Log in to Docker Hub
if: env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != ''
uses: docker/login-action@v3
with:
username: ${{ env.DOCKERHUB_USERNAME }}
password: ${{ env.DOCKERHUB_TOKEN }}
- name: Resolve Docker tags
id: docker_tags
env:
TAG: ${{ needs.beta_version.outputs.tag }}
SUFFIX: ${{ matrix.suffix }}
run: |
{
echo "tags<<EOF"
echo "${GHCR_IMAGE}:${TAG}${SUFFIX}"
if [[ -n "$DOCKERHUB_USERNAME" && -n "$DOCKERHUB_TOKEN" ]]; then
echo "${DOCKERHUB_IMAGE}:${TAG}${SUFFIX}"
fi
echo "EOF"
} >> "$GITHUB_OUTPUT"
if [[ -z "$DOCKERHUB_USERNAME" || -z "$DOCKERHUB_TOKEN" ]]; then
echo "::notice::Docker Hub secrets not configured; publishing GHCR only."
fi
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.docker_tags.outputs.tags }}
build-args: |
ENABLE_OTEL=${{ matrix.enable_otel }}
ENABLE_EMBEDUI=${{ matrix.enable_embedui }}
ENABLE_PYTHON=${{ matrix.enable_python }}
ENABLE_FULL_SKILLS=${{ matrix.enable_full_skills }}
VERSION=${{ needs.beta_version.outputs.tag }}
cache-from: type=gha,scope=dev-beta-${{ matrix.variant }}
cache-to: type=gha,mode=max,scope=dev-beta-${{ matrix.variant }}
promote_beta_aliases:
needs: [beta_version, docker_images]
if: needs.beta_version.outputs.released == 'true'
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
env:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
steps:
- uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Log in to Docker Hub
if: env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != ''
uses: docker/login-action@v3
with:
username: ${{ env.DOCKERHUB_USERNAME }}
password: ${{ env.DOCKERHUB_TOKEN }}
- name: Promote beta aliases
env:
TAG: ${{ needs.beta_version.outputs.tag }}
run: |
docker buildx imagetools create -t "${GHCR_IMAGE}:beta" "${GHCR_IMAGE}:${TAG}"
docker buildx imagetools create -t "${GHCR_IMAGE}:beta-full" "${GHCR_IMAGE}:${TAG}-full"
if [[ -n "$DOCKERHUB_USERNAME" && -n "$DOCKERHUB_TOKEN" ]]; then
docker buildx imagetools create -t "${DOCKERHUB_IMAGE}:beta" "${DOCKERHUB_IMAGE}:${TAG}"
docker buildx imagetools create -t "${DOCKERHUB_IMAGE}:beta-full" "${DOCKERHUB_IMAGE}:${TAG}-full"
else
echo "::notice::Docker Hub secrets not configured; promoted GHCR beta aliases only."
fi
.gitignore
+1
View File
@@ -1,5 +1,6 @@
# Test artifacts
tests/integration/testdata/
backup/
# Binary
openclaw-go
AGENTS.md
+268
View File
@@ -0,0 +1,268 @@
# GoClaw Gateway
PostgreSQL multi-tenant AI agent gateway with WebSocket RPC + HTTP API.
## Language
Always respond in the same language as the user's prompt. If the user writes in Vietnamese, respond in Vietnamese. If in English, respond in English. Match the user's language naturally.
## Tech Stack
**Backend:** Go 1.26, Cobra CLI, gorilla/websocket, pgx/v5 (database/sql, no ORM), golang-migrate, go-rod/rod, telego (Telegram)
**Web UI:** React 19, Vite 6, TypeScript, Tailwind CSS 4, Radix UI, Zustand, React Router 7. Located in `ui/web/`. **Use `pnpm` (not npm).**
**Desktop UI:** React 19, Vite 6, TypeScript, Tailwind CSS 4, Zustand, Framer Motion. Located in `ui/desktop/frontend/`. **Use `pnpm`.**
**Desktop App:** Wails v2 (`//go:build sqliteonly`). Located in `ui/desktop/`. Embeds gateway + React frontend in single binary.
**Database:** PostgreSQL 18 with pgvector (standard). SQLite via `modernc.org/sqlite` (desktop/lite). Raw SQL with `$1, $2` (PG) or `?` (SQLite) positional params. Nullable columns: `*string`, `*time.Time`, etc.
## Project Structure
```
cmd/ CLI commands, gateway startup, onboard wizard, migrations
internal/
├── agent/ Agent loop (think→act→observe), router, resolver, input guard
├── bootstrap/ System prompt files (SOUL.md, IDENTITY.md) + seeding + per-user seed
├── bus/ Event bus system
├── cache/ Caching layer
├── channels/ Channel manager: Telegram, Feishu/Lark, Zalo, Discord, WhatsApp
│ └── whatsapp/ Native WhatsApp via whatsmeow (v3)
├── config/ Config loading (JSON5) + env var overlay
├── consolidation/ Memory consolidation workers (episodic, semantic, dreaming) (v3)
├── crypto/ AES-256-GCM encryption for API keys
├── cron/ Cron scheduling (at/every/cron expr)
├── edition/ Edition system (Lite, Standard) with feature gating
├── eventbus/ Domain event bus with worker pool, dedup, retry (v3)
├── gateway/ WS + HTTP server, client, method router
│ └── methods/ RPC handlers (chat, agents, sessions, config, skills, cron, pairing)
├── hooks/ Hook system for extensibility
├── http/ HTTP API (/v1/chat/completions, /v1/agents, /v1/skills, etc.)
├── i18n/ Message catalog: T(locale, key, args...) + per-locale catalogs (en/vi/zh)
├── knowledgegraph/ Knowledge graph storage and traversal
├── mcp/ Model Context Protocol bridge/server
├── media/ Media handling utilities
├── memory/ Memory system (pgvector)
├── oauth/ OAuth authentication
├── orchestration/ Orchestration primitives: BatchQueue[T] generic, ChildResult, media conversion (v3)
├── permissions/ RBAC (admin/operator/viewer)
├── pipeline/ 8-stage agent pipeline (context→history→prompt→think→act→observe→memory→summarize)
├── providers/ LLM providers: Anthropic (native HTTP+SSE), OpenAI-compat (HTTP+SSE), DashScope (Alibaba Qwen), Codex CLI (stdio+MCP bridge), ACP (Anthropic Console Proxy), Codex (OpenAI), Vertex AI (GCP OAuth2 + OpenAI-compat)
├── providerresolve/ Provider adapter + model registry with forward-compat resolver
├── sandbox/ Docker-based code execution sandbox
├── scheduler/ Lane-based concurrency (main/subagent/cron)
├── sessions/ Session management
├── skills/ SKILL.md loader + BM25 search
├── store/ Store interfaces + implementations (PostgreSQL, SQLite)
│ ├── base/ Shared store abstractions: Dialect interface, helpers (NilStr, BuildMapUpdate, BuildScopeClause)
│ ├── pg/ PostgreSQL implementations (database/sql + pgx/v5)
│ └── sqlitestore/ SQLite implementations (modernc.org/sqlite)
├── tasks/ Task management
├── tokencount/ tiktoken BPE token counting
├── tools/ Tool registry, filesystem, exec, web, memory, subagent, MCP bridge, delegate
├── tracing/ LLM call tracing + optional OTel export (build-tag gated)
├── tts/ Text-to-Speech (OpenAI, ElevenLabs, Edge, MiniMax)
├── updater/ Desktop auto-update checker (Lite edition)
├── upgrade/ Database schema version tracking
├── vault/ Knowledge Vault with wikilinks, hybrid search, FS sync
├── workspace/ WorkspaceContext resolver for 6 scenarios
pkg/protocol/ Wire types (frames, methods, errors, events)
pkg/browser/ Browser automation (Rod + CDP)
migrations/ PostgreSQL migration files
ui/web/ React SPA (pnpm, Vite, Tailwind, Radix UI)
ui/desktop/ Wails v2 desktop app (React frontend + embedded gateway)
```
## Key Patterns
- **Store layer:** Interface-based (`store.SessionStore`, `store.AgentStore`, etc.) with shared Dialect pattern in `store/base/`. PostgreSQL (`pg/`) and SQLite (`sqlitestore/`) implementations use `database/sql` + `pgx/v5/stdlib` + sqlx, raw SQL, `BuildMapUpdate()` and `BuildScopeClause()` helpers
- **Agent types:** `open` (per-user context, 7 files) vs `predefined` (shared context + USER.md per-user)
- **Agent identity:** Dual-identity pattern (agent_key vs UUID) applies to agents, teams, tenants. Rule: UUID for DB/FK/events, agent_key for logs/paths/UI. See `docs/agent-identity-conventions.md`
- **Context files:** `agent_context_files` (agent-level) + `user_context_files` (per-user), routed via `ContextFileInterceptor`
- **Providers:** Anthropic (native HTTP+SSE), OpenAI-compat (HTTP+SSE), DashScope (Alibaba Qwen), Codex CLI (stdio+MCP bridge), ACP (Anthropic Console Proxy), Codex (OpenAI), Vertex AI (GCP OAuth2 service account or ADC + OpenAI-compat endpoint, `internal/providers/vertex.go`). All use `RetryDo()` for retries. Loads from `llm_providers` table with encrypted API keys. ProviderAdapter enables pluggable implementations with ModelRegistry forward-compat resolver. Shared SSEScanner in `providers/sse_reader.go` for streaming providers
- **Pipeline:** 8-stage loop (context→history→prompt→think→act→observe→memory→summarize) with pluggable callbacks, always-on execution path
- **DomainEventBus:** Typed events with worker pool, dedup, retry. Used by consolidation pipeline and memory workers
- **3-tier memory:** Working (conversation) → Episodic (session summaries) → Semantic (KG). Progressive loading L0/L1/L2 with auto-inject for L0
- **Knowledge Vault:** Document registry + [[wikilinks]] + hybrid search, query layer above existing stores, FS sync, unified search
- **Context propagation:** `store.WithAgentType(ctx)`, `store.WithUserID(ctx)`, `store.WithAgentID(ctx)`, `store.WithLocale(ctx)`, `store.WithTenantID(ctx)`
- **Request middleware:** Composable chain (cache, service tier, request guards), zero-alloc fast path for hot operations
- **Self-evolution:** Metrics → suggestions → auto-adapt. 3 progressive stages: metrics collection, suggestion analysis, guardrail-protected apply/rollback
- **Orchestration:** Delegate tool for inter-agent task delegation with agent_links, 3 delegation modes (auto/explicit/manual), token-aware work distribution. BatchQueue[T] generic for result aggregation
- **WebSocket protocol:** Frame types `req`/`res`/`event`. First request must be `connect`
- **Config:** JSON5 at `GOCLAW_CONFIG` env. Secrets in `.env.local` or env vars, never in config.json
- **Security:** Rate limiting, input guard (detection-only), CORS, shell deny patterns, SSRF protection, path traversal prevention, AES-256-GCM encryption. All security logs: `slog.Warn("security.*")`
- **Telegram formatting:** LLM output → `SanitizeAssistantContent()` → `markdownToTelegramHTML()` → `chunkHTML()` → `sendHTML()`. Tables rendered as ASCII in `<pre>` tags
- **i18n:** Web UI uses `i18next` with namespace-split locale files in `ui/web/src/i18n/locales/{lang}/`. Backend uses `internal/i18n` message catalog with `i18n.T(locale, key, args...)`. Locale propagated via `store.WithLocale(ctx)` — WS `connect` param `locale`, HTTP `Accept-Language` header. Supported: en (default), vi, zh. New user-facing strings: add key to `internal/i18n/keys.go`, add translations to all 3 catalog files. New UI strings: add key to all 3 locale dirs. Bootstrap templates (SOUL.md, etc.) stay English-only (LLM consumption).
## Running
```bash
go build -o goclaw . && ./goclaw onboard && source .env.local && ./goclaw
./goclaw migrate up # DB migrations
# Integration tests (requires pgvector pg18 on port 5433)
docker run -d --name pgtest -p 5433:5432 -e POSTGRES_PASSWORD=test -e POSTGRES_DB=goclaw_test pgvector/pgvector:pg18
TEST_DATABASE_URL="postgres://postgres:test@localhost:5433/goclaw_test?sslmode=disable" \
go test -v -tags integration ./tests/integration/
# Layered tests
make test-invariants # P0 - tenant isolation (blocking)
make test-contracts # P1 - API schemas (requires server)
make test-scenarios # P2 - user journeys (requires server)
make test-critical # P0 + P1 (pre-merge)
cd ui/web && pnpm install && pnpm dev # Web dashboard (dev)
# Desktop (Wails + SQLite)
cd ui/desktop && wails dev -tags sqliteonly # Dev mode with hot reload (direct)
make desktop-dev # Same as above via Makefile
make desktop-build VERSION=0.1.0 # Build .app (macOS) or .exe (Windows)
make desktop-dmg VERSION=0.1.0 # Create .dmg installer (macOS only)
```
## CI/CD & Releases
### Workflows
| Workflow | Trigger | Purpose |
|----------|---------|---------|
| `ci.yaml` | push main, PR→main/dev | Go build+test+vet, Web build |
| `release.yaml` | tag `v[0-9]+.[0-9]+.[0-9]+` | Binaries + Docker (4 variants + web) + Discord |
| `release-beta.yaml` | tag `v*-beta*` / `v*-rc*` | Beta binaries + Docker + GitHub prerelease |
| `release-desktop.yaml` | tag `lite-v*` | Desktop app (macOS+Windows), auto prerelease for `-beta`/`-rc` tags |
### Creating Releases
**Standard release** — manual tag push after merging `dev` → `main`:
```bash
git tag v3.0.0 && git push origin v3.0.0
```
**Beta release** (from dev):
```bash
git tag v2.67.0-beta.1 && git push origin v2.67.0-beta.1 # standard beta
git tag lite-v1.2.0-beta.1 && git push origin lite-v1.2.0-beta.1 # lite beta
```
**Desktop release:**
```bash
git tag lite-v1.1.0 && git push origin lite-v1.1.0 # stable
git tag lite-v1.1.0-beta.1 && git push origin lite-v1.1.0-beta.1 # beta (prerelease)
```
### Docker Images
Published to GHCR (`ghcr.io/nextlevelbuilder/goclaw`) and Docker Hub (`digitop/goclaw`).
| Variant | Tag | Contents |
|---------|-----|----------|
| latest | `:latest`, `:vX.Y.Z` | Backend + web UI + Python |
| base | `:base`, `:vX.Y.Z-base` | Backend only, no UI/runtimes |
| full | `:full`, `:vX.Y.Z-full` | All runtimes + skills pre-installed |
| web | `-web:latest` | Standalone web UI (Nginx) |
| beta | `:beta`, `:vX.Y.Z-beta.N` | Beta builds from dev |
OTel and Tailscale variants are not pre-built — build from source with the appropriate `--build-arg ENABLE_OTEL=true` or `-tags tsnet` flag if needed.
### Tag Pattern Safety
- `release.yaml`: tag-triggered (`v[0-9]+.[0-9]+.[0-9]+`) — clean semver only, no beta/rc
- `release-beta.yaml`: tag-triggered (`v*-beta*`, `v*-rc*`) — never matches clean semver
- `release-desktop.yaml`: tag-triggered (`lite-v*`) — `lite-` prefix prevents overlap
- No workflow triggers overlap — each tag pattern is distinct. Merging to `main` only triggers CI, not release
## Desktop Edition (Lite)
- **Build tag:** `//go:build sqliteonly` — desktop binary includes only SQLite, no PostgreSQL
- **Edition system:** `internal/edition/edition.go` — `Lite` preset auto-selected for SQLite backend. Check `edition.Current()` for feature limits
- **Entry point:** `ui/desktop/main.go` + `ui/desktop/app.go` — Wails bindings, embedded gateway
- **Secrets:** OS keyring (`go-keyring`) with file fallback at `~/.goclaw/secrets/`
- **Data dir:** `~/.goclaw/data/` (SQLite DB, configs)
- **Workspace:** `~/.goclaw/workspace/` (agent files, team workspace)
- **Port:** 18790 (localhost only), configurable via `GOCLAW_PORT`
- **WS params:** All WS method params use **camelCase** (`teamId`, `taskId`, `sessionKey`) — match Go struct `json:"..."` tags
- **Version:** `cmd.Version` set via `-ldflags` at build time. Frontend calls `wails.getVersion()`
- **Auto-update:** `internal/updater/updater.go` checks GitHub Releases for `lite-v*` tags. Frontend `UpdateBanner` shows notification
- **Releases:** Tag `lite-v*` triggers `.github/workflows/release-desktop.yaml` → builds macOS (arm64+amd64) + Windows → GitHub Release
- **Install scripts:** `scripts/install-lite.sh` (macOS), `scripts/install-lite.ps1` (Windows PowerShell)
- **Lite limits:** 5 agents, 1 team, 5 members, 50 sessions. No channels, heartbeat, file storage UI, skill self-manage, KG, RBAC, multi-tenant
- **Tool gating:** `TeamActionPolicy` in `internal/tools/team_action_policy.go` — lite blocks comment/review/approve/reject/attach/ask_user. `skill_manage`/`publish_skill` not registered in lite
- **File serving:** 2-layer path isolation in `internal/http/files.go` — workspace boundary (all editions) + tenant scope (standard only with RBAC)
## Plan Verification Rules
Apply before finalizing any multi-phase plan. Trust-but-verify between scout → planner → final plan.
### Verification discipline (what to verify)
1. **Verify factual claims against code** — re-grep/re-count every number, path, endpoint. Don't copy from scout summaries.
2. **Trace semantics, not just cite lines** — when plan references existing/upstream code, identify WHEN each field mutates and under WHAT conditions. Line-range citation without control-flow trace = how ports silently invert behavior. Check: every call, or specific branches only?
3. **No fabricated identifiers / API families** — every symbol in plan must cite `file:line`. RED FLAGS: plausible-sounding wrappers (`Keyring`, `Validator`, `Manager`), centralized packages (`internal/security`, `internal/auth`) that may be scattered, OTel-style (`StartSpan/EndSpan`) when codebase is emit-based. When unsure, `go doc <pkg>` lists actual exported surface. Apply especially when plan says "reuse existing X".
4. **Struct scope audit before adding state** — verify lifetime (per-request/session/agent/process) before adding a field to an existing struct. "Plausibly per-X" is a red flag — grep construction + ownership. Shared-instance state leaks across isolation boundaries.
5. **Gate-premise test math** — before asserting "feature X triggers independently of Y", list all early-returns from function entry to X. Math-verify any fixture claiming "X without Y".
6. **Port = config-shape match** — "faithful port" divergences in config field name/type are silent breaking changes for users copying upstream config. Match upstream shape, or explicitly flag each divergence with rationale in the phase file.
7. **Verify external API endpoints via `docs-seeker`** — before writing endpoint into plan. Sibling APIs often use different roots.
### Scope & coverage (where to look)
8. **Grep delete scope deep** — `grep -rn '<symbol>' .` whole repo. Stubs often have refs in catalogs/routing/switch cases. Enumerate ALL sites in todo.
9. **Signature-change callers enumeration** — grep + list all callers explicitly. "Update all callers" insufficient.
10. **Alias/shim coverage** — enumerate ALL exported symbols via `go doc <pkg>`. Add compile-time signature guards.
11. **Scout desktop and web separately** — `ui/desktop/frontend/` ≠ `ui/web/`. Different structure, i18n namespaces, test framework presence.
### Phasing & ordering (when)
12. **Re-scout on scope change** — if phase promotes from deferred → active, re-scout. Don't reuse brainstorm summary.
13. **Cross-phase gates explicit** — "Phase N-1 merged + tests green" in phase Context. Execution order alone ≠ enforcement.
14. **Zero-coverage characterization test = blocker step** — write byte/request-body fixture test BEFORE migration. Not "recommended".
15. **i18n keys ordering** — add key + 3 catalogs as explicit todo step BEFORE handler code. Missing key = runtime crash.
### Conventions & finalization
16. **Context key style convention** — check existing `context.go` pattern before introducing new key types. Mixed = code smell.
17. **Verify pass MANDATORY after rewrite** — spawn fresh Explore/grep to audit planner output. Don't trust self-validation.
**Pattern to avoid:** user asks → planner writes → report "done".
**Safer pattern:** user asks → scout → planner writes → audit-verify → report.
**Red-team practice:** After planner completes, run `code-reviewer`/`brainstormer` in audit mode: "spot-check 15+ claims vs live codebase". Past catches: fabricated `crypto.Keyring`/`tracing.StartSpan` (agent-hooks plan); inverted TS-port semantics + wrong struct scope + misread early-return gate (context-pruning plan). See `plans/*/reports/audit-*.md` for concrete examples.
## Post-Implementation Checklist
After implementing or modifying Go code, run these checks:
```bash
go fix ./... # Apply Go version upgrades (run before commit)
go build ./... # Compile check (PG build)
go build -tags sqliteonly ./... # Compile check (Desktop/SQLite build)
go vet ./... # Static analysis
go test -race ./tests/integration/ # Integration tests with race detector
```
Go conventions to follow:
- Use `errors.Is(err, sentinel)` instead of `err == sentinel`
- Use `switch/case` instead of `if/else if` chains on the same variable
- Use `append(dst, src...)` instead of loop-based append
- Always handle errors; don't ignore return values
- **Migrations (dual-DB):** PostgreSQL and SQLite have **separate migration systems**. When adding schema changes: (1) PG: add SQL in `migrations/` + bump `RequiredSchemaVersion` in `internal/upgrade/version.go`. (2) SQLite: update `internal/store/sqlitestore/schema.sql` (full schema for fresh DBs) + add incremental patch in `schema.go` `migrations` map + bump `SchemaVersion` constant. **Always update both** — missing SQLite migrations cause desktop edition to crash on startup
- **i18n strings:** When adding user-facing error messages, add key to `internal/i18n/keys.go` and translations to `catalog_en.go`, `catalog_vi.go`, `catalog_zh.go`. For UI strings, add to all locale JSON files in `ui/web/src/i18n/locales/{en,vi,zh}/`
- **SQL safety:** When implementing or modifying SQL store code (`store/pg/*.go`), always verify: (1) All user inputs use parameterized queries (`$1, $2, ...`), never string concatenation — prevents SQL injection. (2) Queries are optimized — no N+1 queries, no unnecessary full table scans. (3) WHERE clauses, JOINs, and ORDER BY columns use existing indices — check migration files for available indexes
- **DB query reuse:** Before adding a new DB query for key entities (teams, agents, sessions, users), check if the same data is already fetched earlier in the current flow/pipeline. Prefer passing resolved data through context, event payloads, or function params rather than re-querying. Duplicate queries waste DB resources and add latency
- **Solution design:** When designing a fix or feature, identify the root cause first — don't just patch symptoms. Think through production scenarios (high concurrency, multi-tenant isolation, failure cascades, long-running sessions) to ensure the solution holds up. Prefer explicit configuration over runtime heuristics. Prefer the simplest solution that addresses the root cause directly
- **Tenant-scope guards on admin writes:** `RoleAdmin` is not a tenant check. Writes to **global** tables (no `tenant_id` column — e.g. `builtin_tools`, disk config, package mgmt) must gate with `http.requireMasterScope` / WS `requireMasterScope(requireOwner(...))`. Writes to **tenant-scoped** tables must gate with `http.requireTenantAdmin` + SQL `WHERE tenant_id = $N`. Shared predicate: `store.IsMasterScope(ctx)`. See `CONTRIBUTING.md` → "Tenant-scope guards" for the full decision table and anti-patterns.
- **Skip load / stress / benchmark tests.** Do NOT write throughput benchmarks, p95/p99 latency assertions, or `runtime.ReadMemStats`-based memory-leak tests for regular feature work. They flake on shared CI runners, waste runner time, and rarely catch real bugs. Only add load tests when explicitly requested for a specific investigation. For normal "prove it works" coverage, use unit + integration + chaos tests.
## Mobile UI/UX Rules
When implementing or modifying web UI components, follow these rules to ensure mobile compatibility:
- **Viewport height:** Use `h-dvh` (dynamic viewport height), never `h-screen`. `h-screen` causes content to hide behind mobile browser chrome and virtual keyboards
- **Input font-size:** All `<input>`, `<textarea>`, `<select>` must use `text-base md:text-sm` (16px on mobile). Font-size < 16px triggers iOS Safari auto-zoom on focus
- **Safe areas:** Root layout must use `viewport-fit=cover` meta tag. Apply `safe-top`, `safe-bottom`, `safe-left`, `safe-right` utility classes on edge-anchored elements (app shell, sidebar, toasts, chat input) for notched devices
- **Touch targets:** Icon buttons must have ≥44px hit area on touch devices. CSS in `index.css` uses `@media (pointer: coarse)` with `::after` pseudo-elements to expand targets
- **Tables:** Always wrap `<table>` in `<div className="overflow-x-auto">` and set `min-w-[600px]` on the table for horizontal scroll on narrow screens
- **Grid layouts:** Use mobile-first responsive grids: `grid-cols-1 sm:grid-cols-2 lg:grid-cols-N`. Never use fixed `grid-cols-N` without a mobile breakpoint
- **Dialogs:** Full-screen on mobile with slide-up animation (`max-sm:inset-0`), centered with zoom on desktop (`sm:max-w-lg`). Handled in `ui/dialog.tsx`
- **Virtual keyboard:** Chat input uses `useVirtualKeyboard()` hook + `var(--keyboard-height, 0px)` CSS var to stay above the keyboard
- **Scroll behavior:** Use `overscroll-contain` on scrollable areas to prevent background scroll. Auto-scroll: smooth for incoming messages, instant on user send
- **Landscape:** Use `landscape-compact` class on top bars to reduce padding in phone landscape orientation (`max-height: 500px`)
- **Portal dropdowns in dialogs:** Custom dropdown components using `createPortal(content, document.body)` MUST add `pointer-events-auto` class to the dropdown element. Radix Dialog sets `pointer-events: none` on `document.body` — without this class, dropdowns are unclickable. Radix-native portals (Select, Popover) handle this automatically
- **Timezone:** User timezone stored in Zustand (`useUiStore`). Charts use `formatBucketTz()` from `lib/format.ts` with native `Intl.DateTimeFormat` — no date-fns-tz dependency
- **ErrorBoundary key:** `AppLayout` uses `<ErrorBoundary key={stableErrorBoundaryKey(pathname)}>` which strips dynamic segments (`/chat/session-A` → `/chat`). NEVER use `key={location.pathname}` on ErrorBoundary/Suspense wrapping `<Outlet>` — it causes full page remount on param changes. Pages with sub-navigation (chat sessions, detail pages) must share a stable key
- **Route params as source of truth:** For pages with URL params (e.g. `/chat/:sessionKey`), derive state from `useParams()` — do NOT duplicate into `useState`. Dual state causes race conditions between `setState` and `navigate()` leading to UI flash (state bounces: B→A→B). Use optional params (`/chat/:sessionKey?`) instead of two separate routes
CHANGELOG.md
+53
View File
@@ -4,6 +4,46 @@ All notable changes to GoClaw are documented here. For full documentation, see [
## Unreleased
### Added
- **Skill agent manage grants** — Adds per-agent skill edit/delete grants with
backend checks, HTTP/WS support, SQLite and PostgreSQL schema updates, and web
dashboard controls for granting and revoking manage access.
- **Packages Update Flow (Phase 2a: pip + npm)** — closes #900 (Phase 2a). Extends
Phase 1 update infrastructure to pip and npm package sources. `/v1/packages/updates`
now returns mixed-source results with an `availability: {github, pip, npm}` map.
Multi-source UI with per-source filter pills; unavailable sources (binary not on PATH
or Lite edition) hidden automatically. apk deferred to Phase 2b.
See `docs/packages-pip-npm.md` for command matrix, runbook, and min versions.
- **Packages Update Flow (Phase 1: GitHub binaries)** — closes #900. Proactive
"N updates available" badge + per-row `[Update]` + `[Update All]` on the
Runtime & Packages page. Backend endpoints under `/v1/packages/updates*`
(master-scope). ETag-aware polling (304 responses don't burn rate limit),
stale-while-revalidate cache, atomic two-phase `.bak` swap with rollback.
Pre-release detection via regex + GitHub API flag; semver ordering via
`golang.org/x/mod/semver`; non-semver tags use string-inequality fallback
with downgrade protection. WebSocket events `package.update.*` for owner
clients. See `docs/packages-github.md` § "Updating Installed Packages".
### Changed
- **ChatGPT Subscription (OAuth)** — default model and backend-owned model catalog
now prefer `gpt-5.5`, with reasoning metadata and context-window defaults updated
for provider-first model selection.
### Fixed
- **Upstream critical security remediation** — hardens gateway no-token fallback,
Feishu/Lark and Pancake webhooks, sandbox path/write handling, tenant-admin
checks for mutable HTTP surfaces, and Lite hook schema migration verification.
- **SecureCLI runtime npm binaries** — binary discovery and credentialed exec now
resolve tools installed under the GoClaw runtime directories, including
`{runtimeDir}/npm-global/bin`, and support single-binary npm package aliases
such as `openrouter-cli` exposing `orc`.
### Breaking Changes
- **Context pruning now opt-in.** Previously tool-result trimming ran by default
@@ -50,6 +90,18 @@ All notable changes to GoClaw are documented here. For full documentation, see [
`display_name` across the inbound → outbound hop so the private-reply
template variables survive the agent pipeline round-trip.
### Fixed
- **Skill grant tenant isolation.** Agent skill grants now validate both the
skill and agent tenant scope before insert, revoke, grant listing, or
can-manage checks. Visibility auto-promote/auto-demote updates are scoped to
the calling tenant or system skills so one tenant cannot mutate another
tenant's skill.
- **Agent provider switching.** Saving an agent after changing provider/model now
handles cleared ChatGPT OAuth routing config without writing SQL NULL into
NOT NULL JSON config columns.
## Project Status
### Implemented & Tested in Production
@@ -82,6 +134,7 @@ All notable changes to GoClaw are documented here. For full documentation, see [
- **Hooks system** — Event-driven hooks with command evaluators (shell exit code) and agent evaluators (delegate to reviewer). Blocking gates with auto-retry and recursion-safe evaluation.
- **Media tools** — `create_image` (DashScope, MiniMax), `create_audio` (OpenAI, ElevenLabs, MiniMax, Suno), `create_video` (MiniMax, Veo), `read_document` (Gemini File API), `read_image`, `read_audio`, `read_video`. Persistent media storage with lazy-loaded MediaRef.
- **Additional provider modes** — Claude CLI (Anthropic via stdio + MCP bridge), Codex (OpenAI gpt-5.3-codex via OAuth).
- **Google Cloud Vertex AI provider** — Enterprise GCP integration via Vertex OpenAI-compatible endpoint. OAuth2 service account auth (inline JSON or file path) with automatic token refresh, plus Application Default Credentials (ADC) for GKE/Cloud Run/Compute Engine. Regional endpoints for data residency (e.g. `asia-southeast1`, `us-central1`). Addresses [#576](https://github.com/nextlevelbuilder/goclaw/issues/576).
- **Knowledge graph** — LLM-powered entity extraction, graph traversal, force-directed visualization, and `knowledge_graph_search` agent tool.
- **Memory management** — Admin dashboard for memory documents (CRUD, semantic search, chunk/embedding details, bulk re-indexing).
- **Persistent pending messages** — Channel messages persisted to PostgreSQL with auto-compaction (LLM summarization) and monitoring dashboard.
CLAUDE.md
+7 -4
View File
@@ -44,7 +44,7 @@ internal/
├── orchestration/ Orchestration primitives: BatchQueue[T] generic, ChildResult, media conversion (v3)
├── permissions/ RBAC (admin/operator/viewer)
├── pipeline/ 8-stage agent pipeline (context→history→prompt→think→act→observe→memory→summarize)
├── providers/ LLM providers: Anthropic (native HTTP+SSE), OpenAI-compat (HTTP+SSE), DashScope (Alibaba Qwen), Claude CLI (stdio+MCP bridge), ACP (Anthropic Console Proxy), Codex (OpenAI)
├── providers/ LLM providers: Anthropic (native HTTP+SSE), OpenAI-compat (HTTP+SSE), DashScope (Alibaba Qwen), Claude CLI (stdio+MCP bridge), ACP (Anthropic Console Proxy), Codex (OpenAI), Vertex AI (GCP OAuth2 + OpenAI-compat)
├── providerresolve/ Provider adapter + model registry with forward-compat resolver
├── sandbox/ Docker-based code execution sandbox
├── scheduler/ Lane-based concurrency (main/subagent/cron)
@@ -76,7 +76,7 @@ ui/desktop/ Wails v2 desktop app (React frontend + embedded ga
- **Agent types:** `open` (per-user context, 7 files) vs `predefined` (shared context + USER.md per-user)
- **Agent identity:** Dual-identity pattern (agent_key vs UUID) applies to agents, teams, tenants. Rule: UUID for DB/FK/events, agent_key for logs/paths/UI. See `docs/agent-identity-conventions.md`
- **Context files:** `agent_context_files` (agent-level) + `user_context_files` (per-user), routed via `ContextFileInterceptor`
- **Providers:** Anthropic (native HTTP+SSE), OpenAI-compat (HTTP+SSE), DashScope (Alibaba Qwen), Claude CLI (stdio+MCP bridge), ACP (Anthropic Console Proxy), Codex (OpenAI). All use `RetryDo()` for retries. Loads from `llm_providers` table with encrypted API keys. ProviderAdapter enables pluggable implementations with ModelRegistry forward-compat resolver. Shared SSEScanner in `providers/sse_reader.go` for streaming providers
- **Providers:** Anthropic (native HTTP+SSE), OpenAI-compat (HTTP+SSE), DashScope (Alibaba Qwen), Claude CLI (stdio+MCP bridge), ACP (Anthropic Console Proxy), Codex (OpenAI), Vertex AI (GCP OAuth2 service account or ADC + OpenAI-compat endpoint, `internal/providers/vertex.go`). All use `RetryDo()` for retries. Loads from `llm_providers` table with encrypted API keys. ProviderAdapter enables pluggable implementations with ModelRegistry forward-compat resolver. Shared SSEScanner in `providers/sse_reader.go` for streaming providers
- **Pipeline:** 8-stage loop (context→history→prompt→think→act→observe→memory→summarize) with pluggable callbacks, always-on execution path
- **DomainEventBus:** Typed events with worker pool, dedup, retry. Used by consolidation pipeline and memory workers
- **3-tier memory:** Working (conversation) → Episodic (session summaries) → Semantic (KG). Progressive loading L0/L1/L2 with auto-inject for L0
@@ -123,6 +123,7 @@ make desktop-dmg VERSION=0.1.0 # Create .dmg installer (macOS only
| Workflow | Trigger | Purpose |
|----------|---------|---------|
| `ci.yaml` | push main, PR→main/dev | Go build+test+vet, Web build |
| `dev-beta-release.yaml` | push dev | Go build+test+vet, Web build, semantic beta prerelease, beta Docker |
| `release.yaml` | tag `v[0-9]+.[0-9]+.[0-9]+` | Binaries + Docker (4 variants + web) + Discord |
| `release-beta.yaml` | tag `v*-beta*` / `v*-rc*` | Beta binaries + Docker + GitHub prerelease |
| `release-desktop.yaml` | tag `lite-v*` | Desktop app (macOS+Windows), auto prerelease for `-beta`/`-rc` tags |
@@ -136,7 +137,8 @@ git tag v3.0.0 && git push origin v3.0.0
**Beta release** (from dev):
```bash
git tag v2.67.0-beta.1 && git push origin v2.67.0-beta.1 # standard beta
git push origin dev # auto beta via CI
git tag v2.67.0-beta.1 && git push origin v2.67.0-beta.1 # manual/backfill beta
git tag lite-v1.2.0-beta.1 && git push origin lite-v1.2.0-beta.1 # lite beta
```
@@ -163,9 +165,10 @@ OTel and Tailscale variants are not pre-built — build from source with the app
### Tag Pattern Safety
- `release.yaml`: tag-triggered (`v[0-9]+.[0-9]+.[0-9]+`) — clean semver only, no beta/rc
- `dev-beta-release.yaml`: branch-triggered on `dev`; creates `vX.Y.Z-beta.N` tags after CI passes
- `release-beta.yaml`: tag-triggered (`v*-beta*`, `v*-rc*`) — never matches clean semver
- `release-desktop.yaml`: tag-triggered (`lite-v*`) — `lite-` prefix prevents overlap
- No workflow triggers overlap — each tag pattern is distinct. Merging to `main` only triggers CI, not release
- Stable and desktop tag patterns remain distinct. `dev` branch pushes create beta releases only after CI passes
## Desktop Edition (Lite)
README.md
+24
View File
@@ -292,6 +292,30 @@ Typed domain events power the consolidation pipeline — session summaries, know
> Full tool reference at [docs.goclaw.sh](https://docs.goclaw.sh/#custom-tools)
## Webhook API
Trigger agents or send channel messages from external systems without the gateway token.
```bash
# Bearer auth — sync LLM call
curl -X POST https://example.com/v1/webhooks/llm \
-H "Authorization: Bearer wh_..." \
-H "Content-Type: application/json" \
-d '{"input":"Summarize today metrics","mode":"sync"}'
# HMAC auth — sign with hmac_signing_key from create response
TS=$(date +%s); BODY='{"input":"hi","mode":"sync"}'
SIG=$(echo -n "${TS}.${BODY}" | openssl dgst -sha256 -mac HMAC \
-macopt "hexkey:${WEBHOOK_HMAC_KEY}" | awk '{print $2}')
curl -X POST https://example.com/v1/webhooks/llm \
-H "Content-Type: application/json" \
-H "X-Webhook-Id: ${WEBHOOK_ID}" \
-H "X-GoClaw-Signature: t=${TS},v1=${SIG}" \
-d "$BODY"
```
See **[docs/webhooks.md](docs/webhooks.md)** for the full reference: auth, async callbacks, retry schedule, HMAC examples, channel matrix.
## Documentation
Full documentation at **[docs.goclaw.sh](https://docs.goclaw.sh)** — or browse the source in [`goclaw-docs/`](https://github.com/nextlevelbuilder/goclaw-docs)
cmd/auth.go
+1 -1
View File
@@ -45,7 +45,7 @@ func authStatusCmd() *cobra.Command {
name = provider
}
fmt.Printf("ChatGPT OAuth account: active (alias: %s)\n", name)
fmt.Printf("Use model prefix '%s/' in agent config (e.g. %s/gpt-5.4).\n", name, name)
fmt.Printf("Use model prefix '%s/' in agent config (e.g. %s/gpt-5.5).\n", name, name)
} else {
fmt.Printf("No ChatGPT OAuth tokens found for alias '%s'.\n", provider)
fmt.Println("Use the web UI to authenticate this ChatGPT OAuth account.")
cmd/gateway.go
+33 -6
View File
@@ -17,24 +17,24 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/cache"
"github.com/nextlevelbuilder/goclaw/internal/channels"
"github.com/nextlevelbuilder/goclaw/internal/consolidation"
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
kg "github.com/nextlevelbuilder/goclaw/internal/knowledgegraph"
"github.com/nextlevelbuilder/goclaw/internal/channels/discord"
"github.com/nextlevelbuilder/goclaw/internal/channels/facebook"
"github.com/nextlevelbuilder/goclaw/internal/channels/pancake"
"github.com/nextlevelbuilder/goclaw/internal/channels/feishu"
"github.com/nextlevelbuilder/goclaw/internal/channels/pancake"
slackchannel "github.com/nextlevelbuilder/goclaw/internal/channels/slack"
"github.com/nextlevelbuilder/goclaw/internal/channels/telegram"
"github.com/nextlevelbuilder/goclaw/internal/channels/whatsapp"
"github.com/nextlevelbuilder/goclaw/internal/channels/zalo"
zalopersonal "github.com/nextlevelbuilder/goclaw/internal/channels/zalo/personal"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/consolidation"
"github.com/nextlevelbuilder/goclaw/internal/edition"
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
"github.com/nextlevelbuilder/goclaw/internal/gateway"
"github.com/nextlevelbuilder/goclaw/internal/gateway/methods"
"github.com/nextlevelbuilder/goclaw/internal/hooks"
httpapi "github.com/nextlevelbuilder/goclaw/internal/http"
kg "github.com/nextlevelbuilder/goclaw/internal/knowledgegraph"
mcpbridge "github.com/nextlevelbuilder/goclaw/internal/mcp"
"github.com/nextlevelbuilder/goclaw/internal/media"
"github.com/nextlevelbuilder/goclaw/internal/providers"
@@ -44,6 +44,9 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/tools"
"github.com/nextlevelbuilder/goclaw/internal/vault"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
// Register workstation backend factories via init().
_ "github.com/nextlevelbuilder/goclaw/internal/workstation/backends"
)
func runGateway() {
@@ -81,6 +84,10 @@ func runGateway() {
slog.Error("failed to load config", "error", err)
os.Exit(1)
}
if err := config.ValidateGatewayAuth(cfg.Gateway); err != nil {
slog.Error("unsafe gateway auth configuration", "error", err)
os.Exit(1)
}
// Edition override: explicit GOCLAW_EDITION takes precedence over auto-detection.
// Auto-detection happens later in setupStoresAndTracing (sqlite → lite).
@@ -273,6 +280,11 @@ func runGateway() {
// Register cron/heartbeat/session/message tools, aliases, allow-paths, store wiring.
heartbeatTool, hasMemory := wireExtraTools(pgStores, toolsReg, msgBus, workspace, dataDir, agentCfg, globalSkillsDir, builtinSkillsDir)
// Register workstation_exec + claude_remote tools (Standard edition only; deny-all until Phase 6).
// cleanupWorkstation stops the activity sink retention goroutine and drains the write buffer.
cleanupWorkstation := wireWorkstationTools(pgStores, toolsReg, domainBus)
defer cleanupWorkstation()
// Create all agents — resolved lazily from database by the managed resolver.
agentRouter := agent.NewRouter()
if traceCollector != nil {
@@ -317,8 +329,8 @@ func runGateway() {
agentRouter: agentRouter,
toolsReg: toolsReg,
skillsLoader: skillsLoader,
enrichProgress: enrichProgress,
enrichWorker: enrichWorker,
enrichProgress: enrichProgress,
enrichWorker: enrichWorker,
workspace: workspace,
dataDir: dataDir,
domainBus: domainBus,
@@ -331,6 +343,7 @@ func runGateway() {
mcpToolLister = mcpMgr
}
httpapi.InitGatewayToken(cfg.Gateway.Token)
httpapi.InitGatewayNoAuthFallbackAllowed(config.GatewayNoAuthFallbackAllowed(cfg.Gateway))
exportTokenStore := httpapi.InitExportTokenStore()
defer exportTokenStore.Stop()
agentsH, skillsH, tracesH, mcpH, channelInstancesH, providersH, builtinToolsH, pendingMessagesH, teamEventsH, secureCLIH, secureCLIGrantH, mcpUserCredsH := wireHTTP(pgStores, cfg.Agents.Defaults.Workspace, dataDir, bundledSkillsDir, msgBus, toolsReg, providerRegistry, modelReg, permPE.IsOwner, gatewayAddr, mcpToolLister)
@@ -403,6 +416,20 @@ func runGateway() {
slog.Info("registered hooks RPC methods")
}
// Workstations WS methods — Standard edition only.
// Lite (desktop/SQLite) must NOT expose workstation RPC methods.
if edition.Current().Name != "lite" && pgStores.Workstations != nil && pgStores.WorkstationLinks != nil {
wsMethods := methods.NewWorkstationsMethods(pgStores.Workstations, pgStores.WorkstationLinks)
if pgStores.WorkstationPermissions != nil {
wsMethods.SetPermStore(pgStores.WorkstationPermissions)
}
if pgStores.WorkstationActivity != nil {
wsMethods.SetActivityStore(pgStores.WorkstationActivity)
}
wsMethods.Register(server.Router())
slog.Info("registered workstations RPC methods")
}
// Wire post-turn processor for team task dispatch (WS chat.send + HTTP API paths).
if postTurn != nil {
chatMethods.SetPostTurnProcessor(postTurn)
cmd/gateway_builtin_tools.go
+1
View File
@@ -24,6 +24,7 @@ func builtinToolSeedData() []store.BuiltinToolDef {
{Name: "exec", DisplayName: "Execute Command", Description: "Execute a shell command in the workspace and return stdout/stderr", Category: "runtime", Enabled: true,
Metadata: json.RawMessage(`{"config_hint":"Config → Tools → Exec Approval"}`),
},
{Name: "wait", DisplayName: "Wait", Description: "Pause the current agent tool sequence for a bounded number of milliseconds", Category: "runtime", Enabled: true},
// web
{Name: "web_search", DisplayName: "Web Search", Description: "Search the web for information using a search engine (Brave or DuckDuckGo)", Category: "web", Enabled: true,
cmd/gateway_builtin_tools_test.go
+20
View File
@@ -0,0 +1,20 @@
package cmd
import "testing"
func TestBuiltinToolSeedDataIncludesWait(t *testing.T) {
t.Parallel()
for _, def := range builtinToolSeedData() {
if def.Name != "wait" {
continue
}
if def.Category != "runtime" {
t.Fatalf("wait category = %q, want runtime", def.Category)
}
if !def.Enabled {
t.Fatal("wait should be enabled by default")
}
return
}
t.Fatal("builtinToolSeedData() missing wait")
}
cmd/gateway_github_installer.go
+1 -1
View File
@@ -18,7 +18,7 @@ import (
// GOCLAW_PACKAGES_GITHUB_TOKEN optional PAT (boosts rate limit, enables private repos)
// GOCLAW_PACKAGES_MAX_ASSET_SIZE_MB default 200
// GOCLAW_PACKAGES_GITHUB_ALLOWED_ORGS comma-separated allowlist (empty = all allowed)
// GOCLAW_PACKAGES_GITHUB_BIN_DIR default /app/data/.runtime/bin
// GOCLAW_PACKAGES_GITHUB_BIN_DIR default {runtimeDir}/bin
// GOCLAW_PACKAGES_GITHUB_MANIFEST default {BIN_DIR}/../github-packages.json
func initGitHubInstaller() {
cfg := &skills.GitHubPackagesConfig{
cmd/gateway_http_handlers.go
+1 -1
View File
@@ -96,7 +96,7 @@ func wireHTTP(stores *store.Stores, defaultWorkspace, dataDir, bundledSkillsDir
secureCLIH = httpapi.NewSecureCLIHandler(stores.SecureCLI, msgBus)
}
if stores != nil && stores.SecureCLIGrants != nil {
secureCLIGrantH = httpapi.NewSecureCLIGrantHandler(stores.SecureCLIGrants, msgBus)
secureCLIGrantH = httpapi.NewSecureCLIGrantHandler(stores.SecureCLIGrants, stores.Tenants, msgBus)
}
return agentsH, skillsH, tracesH, mcpH, channelInstancesH, providersH, builtinToolsH, pendingMessagesH, teamEventsH, secureCLIH, secureCLIGrantH, mcpUserCredsH
cmd/gateway_http_wiring.go
+92 -3
View File
@@ -3,10 +3,12 @@ package cmd
import (
"context"
"log/slog"
"os"
"time"
"github.com/nextlevelbuilder/goclaw/internal/audio"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/edition"
"github.com/nextlevelbuilder/goclaw/internal/gateway/methods"
httpapi "github.com/nextlevelbuilder/goclaw/internal/http"
mcpbridge "github.com/nextlevelbuilder/goclaw/internal/mcp"
@@ -134,8 +136,11 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer(
}
// Runtime package management (install/uninstall system/pip/npm/github packages)
// Wire the update registry AFTER initGitHubInstaller so DefaultGitHubInstaller() is set.
initGitHubInstaller()
d.server.SetPackagesHandler(httpapi.NewPackagesHandler())
pkgHandler := wirePackagesHandler(d)
d.server.SetPackagesHandler(pkgHandler)
d.server.SetGatewayUpgradeHandler(httpapi.NewGatewayUpgradeHandlerFromEnv())
// API documentation (OpenAPI spec + Swagger UI at /docs)
d.server.SetDocsHandler(httpapi.NewDocsHandler())
@@ -149,6 +154,71 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer(
httpapi.InitAPIKeyCache(d.pgStores.APIKeys, d.msgBus)
}
// K10: single shared webhookLimiter — one per process enforces per-tenant RPM cap across
// both LLM and message endpoints. Two separate instances would double the effective cap.
webhookEncKey := os.Getenv("GOCLAW_ENCRYPTION_KEY")
// K6: refuse to mount any webhook handler when GOCLAW_ENCRYPTION_KEY is unset.
// crypto.Encrypt("", "") returns plaintext unchanged, so an empty key would silently
// persist raw secrets to the database — defeating the stated DB-leak protection.
// Skip-mount approach: process still starts (all other subsystems work), but
// /v1/webhooks/* returns 404. Set GOCLAW_ENCRYPTION_KEY to re-enable webhooks.
if webhookEncKey == "" {
slog.Error("webhook subsystem disabled: GOCLAW_ENCRYPTION_KEY not set. Set the env var to enable /v1/webhooks/* endpoints.")
} else {
sharedWebhookLimiter := httpapi.NewWebhookLimiter()
// Webhook admin CRUD — available in all editions (Standard + Lite).
// Runtime routes (/v1/webhooks/message, /v1/webhooks/llm) are mounted by phases 05/06.
if d.pgStores != nil && d.pgStores.Webhooks != nil {
adminH := httpapi.NewWebhooksAdminHandler(
d.pgStores.Webhooks,
d.pgStores.Tenants,
d.msgBus,
)
adminH.SetEncKey(webhookEncKey)
d.server.SetWebhooksAdminHandler(adminH)
}
// Webhook message endpoint — Standard edition only (channels required).
// Phase 05b: POST /v1/webhooks/message → sync channel send (text + optional media).
if edition.Current().AllowsChannels() &&
d.pgStores != nil &&
d.pgStores.Webhooks != nil &&
d.pgStores.WebhookCalls != nil &&
d.pgStores.ChannelInstances != nil &&
d.channelMgr != nil {
msgH := httpapi.NewWebhookMessageHandler(
d.channelMgr,
d.pgStores.ChannelInstances,
d.pgStores.WebhookCalls,
d.pgStores.Webhooks,
sharedWebhookLimiter, // K10: shared limiter
)
msgH.SetEncKey(webhookEncKey) // K6: decrypt secret at HMAC verify time
d.server.SetWebhookMessageHandler(msgH)
}
// Webhook LLM endpoint — all editions (Standard + Lite).
// Phase 06: POST /v1/webhooks/llm → sync agent run (≤30s) or async enqueue.
// LocalhostOnly enforcement is handled by WebhookAuthMiddleware at request time.
// lane=nil → handler self-creates internal default lane (4-slot).
if d.pgStores != nil &&
d.pgStores.Webhooks != nil &&
d.pgStores.WebhookCalls != nil &&
d.agentRouter != nil {
llmH := httpapi.NewWebhookLLMHandler(
d.agentRouter,
d.pgStores.WebhookCalls,
d.pgStores.Webhooks,
sharedWebhookLimiter, // K10: shared limiter
nil, // lane: nil → internal default (4-slot); configurable in future via cfg
)
llmH.SetEncKey(webhookEncKey) // K6: decrypt secret at HMAC verify time
d.server.SetWebhookLLMHandler(llmH)
}
}
// Allow browser-paired users to access HTTP APIs
if d.pgStores.Pairing != nil {
httpapi.InitPairingAuth(d.pgStores.Pairing)
@@ -216,7 +286,7 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer(
d.server.SetFilesHandler(httpapi.NewFilesHandler(d.workspace, d.dataDir))
// Storage file management — browse/delete files under the resolved workspace directory.
d.server.SetStorageHandler(httpapi.NewStorageHandler(d.workspace))
d.server.SetStorageHandler(httpapi.NewStorageHandler(d.workspace, d.pgStores.Tenants))
// Media upload endpoint — accepts multipart file uploads, returns temp path + MIME type.
d.server.SetMediaUploadHandler(httpapi.NewMediaUploadHandler())
@@ -266,7 +336,26 @@ func (d *gatewayDeps) wireHTTPHandlersOnServer(
// Per-tenant TTS config endpoint — allows tenant admins to configure TTS.
if d.pgStores.SystemConfigs != nil && d.pgStores.ConfigSecrets != nil {
d.server.SetTTSConfigHandler(httpapi.NewTTSConfigHandler(d.pgStores.SystemConfigs, d.pgStores.ConfigSecrets))
d.server.SetTTSConfigHandler(httpapi.NewTTSConfigHandler(d.pgStores.SystemConfigs, d.pgStores.ConfigSecrets, d.pgStores.Tenants))
}
// Workstations API — Standard edition only.
// Lite edition MUST NOT expose these routes (silent orphan data + contract violation).
if edition.Current().Name != "lite" {
if d.pgStores != nil && d.pgStores.Workstations != nil && d.pgStores.WorkstationLinks != nil {
wsH := httpapi.NewWorkstationsHandler(
d.pgStores.Workstations,
d.pgStores.WorkstationLinks,
d.pgStores.Tenants,
)
if d.pgStores.WorkstationPermissions != nil {
wsH.SetPermStore(d.pgStores.WorkstationPermissions)
}
if d.pgStores.WorkstationActivity != nil {
wsH.SetActivityStore(d.pgStores.WorkstationActivity)
}
d.server.SetWorkstationsHandler(wsH)
}
}
// Seed + apply builtin tool disables
cmd/gateway_lifecycle.go
+38
View File
@@ -18,6 +18,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tasks"
"github.com/nextlevelbuilder/goclaw/internal/tools"
"github.com/nextlevelbuilder/goclaw/internal/webhooks"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
@@ -141,6 +142,38 @@ func (d *gatewayDeps) runLifecycle(
go consumeInboundMessages(ctx, d.msgBus, d.agentRouter, d.cfg, deps.sched, d.channelMgr, deps.consumerTeamStore, deps.quotaChecker, d.pgStores.Sessions, d.pgStores.Agents, contactCollector, deps.postTurn, deps.subagentMgr)
// Webhook callback worker — delivers async webhook_calls rows to receiver callback_url.
// Runs in both editions: Standard (PG, concurrency=4) and Lite (SQLite, concurrency=1).
// sqliteonly: single callback worker — SQLite lacks SKIP LOCKED; BEGIN IMMEDIATE serializes.
var webhookWorkerCancel context.CancelFunc
if d.pgStores != nil &&
d.pgStores.WebhookCalls != nil &&
d.pgStores.Webhooks != nil &&
d.pgStores.Tenants != nil &&
d.agentRouter != nil {
workerConcurrency := 4
if edition.Current().IsLimited() {
// sqliteonly: single callback worker — SQLite lacks SKIP LOCKED; BEGIN IMMEDIATE serializes.
workerConcurrency = 1
}
ww := webhooks.NewWebhookWorker(
d.pgStores.WebhookCalls,
d.pgStores.Webhooks,
d.pgStores.Tenants,
d.agentRouter,
nil, // limiter: created internally with default per-tenant cap (4)
webhooks.WorkerConfig{
WorkerConcurrency: workerConcurrency,
PerTenantConcurrency: 4,
},
)
// K6: decrypt raw secret for outbound HMAC signing using the same key as inbound verify.
ww.SetEncKey(os.Getenv("GOCLAW_ENCRYPTION_KEY"))
var workerCtx context.Context
workerCtx, webhookWorkerCancel = context.WithCancel(ctx)
go ww.Run(workerCtx)
}
// Task recovery ticker: re-dispatches stale/pending team tasks on startup and periodically.
var taskTicker *tasks.TaskTicker
if d.pgStores.Teams != nil {
@@ -163,6 +196,11 @@ func (d *gatewayDeps) runLifecycle(
taskTicker.Stop()
}
// Stop webhook callback worker — signals Run() to drain in-flight and exit.
if webhookWorkerCancel != nil {
webhookWorkerCancel()
}
// Drain audit log queue before closing DB
if deps.auditCh != nil {
close(deps.auditCh)
cmd/gateway_packages_wiring.go
+87
View File
@@ -0,0 +1,87 @@
package cmd
import (
"log/slog"
"path/filepath"
"github.com/nextlevelbuilder/goclaw/internal/edition"
httpapi "github.com/nextlevelbuilder/goclaw/internal/http"
"github.com/nextlevelbuilder/goclaw/internal/skills"
)
// wirePackagesHandler constructs the UpdateRegistry and wires it into
// PackagesHandler together with the gateway's event publisher.
//
// Called after initGitHubInstaller() so DefaultGitHubInstaller() is non-nil.
// If the installer is not configured (e.g. in integration-test stubs), returns
// a handler with nil registry — the update endpoints return 503.
func wirePackagesHandler(d *gatewayDeps) *httpapi.PackagesHandler {
installer := skills.DefaultGitHubInstaller()
if installer == nil {
slog.Warn("packages: github installer not configured; update endpoints disabled")
return httpapi.NewPackagesHandler(nil, d.msgBus)
}
// Cache file lives next to the manifest dir so it shares the same atomic-
// write guarantees on the same filesystem (no cross-device rename risk).
cachePath := filepath.Join(filepath.Dir(installer.Config.ManifestPath), "updates-cache.json")
cache, err := skills.LoadUpdateCache(cachePath)
if err != nil {
// ErrUpdateCacheCorrupt — log and proceed with an empty cache; a
// background refresh will repopulate on first GET /v1/packages/updates.
slog.Warn("packages: update cache corrupt; starting fresh", "path", cachePath, "error", err)
}
ttl := d.cfg.Packages.UpdatesCheckTTLDuration()
registry := skills.NewUpdateRegistry(cache, cachePath, ttl)
// Share the installer's locker so Install and Update share per-package locks.
registry.Locker = installer.Locker
skills.SetSharedPackageLocker(registry.Locker)
// Register checker + executor for "github" source.
registry.RegisterChecker(skills.NewGitHubUpdateChecker(installer))
executor := skills.NewGitHubUpdateExecutor(installer)
if d.cfg.Packages.ScratchDir != "" {
executor.ScratchDir = d.cfg.Packages.ScratchDir
}
registry.RegisterExecutor(executor)
// Register pip + npm checkers/executors when the edition supports them.
if edition.Current().SupportsPipNpm {
registry.RegisterChecker(skills.NewPipUpdateChecker())
registry.RegisterExecutor(skills.NewPipUpdateExecutor())
registry.RegisterChecker(skills.NewNpmUpdateChecker())
registry.RegisterExecutor(skills.NewNpmUpdateExecutor())
}
// Register apk checker/executor when edition + runtime both permit.
// Double gate: edition flag (compile-time) + /etc/alpine-release (runtime).
// Rationale: Standard-Debian variants pass the edition gate but fail runtime;
// Lite on Alpine fails the edition gate but passes runtime. Both must hold.
if edition.Current().SupportsApk && skills.IsAlpineRuntime() {
registry.RegisterChecker(skills.NewApkUpdateChecker())
registry.RegisterExecutor(skills.NewApkUpdateExecutor())
slog.Info("packages: apk updates registered")
} else if edition.Current().SupportsApk {
// Standard edition but non-Alpine host: emit explicit availability=false
// so frontend can distinguish "not applicable to this runtime" from
// "checker errored". Lite skips both branches → availability.apk absent.
registry.SetAvailability("apk", false)
slog.Info("packages: apk updates skipped (non-Alpine runtime)",
"is_alpine_runtime", skills.IsAlpineRuntime())
} else {
// Lite edition: no registration, no availability seed (key absent in response).
slog.Info("packages: apk updates skipped (edition does not support apk)")
}
slog.Info("packages: update registry wired",
"cache", cachePath,
"ttl", ttl,
"sources", registry.Sources(),
)
return httpapi.NewPackagesHandler(registry, d.msgBus)
}
cmd/gateway_providers.go
+45
View File
@@ -174,6 +174,27 @@ func registerProviders(registry *providers.Registry, cfg *config.Config, modelRe
slog.Info("registered provider", "name", "byteplus-coding")
}
// Google Cloud Vertex AI — OAuth2 service account or Application Default Credentials.
// Registers when project_id + region are set. Credential sources (priority order):
// inline JSON (APIKey) → file path (CredentialsFile) → ADC.
if cfg.Providers.Vertex.ProjectID != "" && cfg.Providers.Vertex.Region != "" {
vcfg := providers.VertexConfig{
Name: "vertex",
CredentialsJSON: cfg.Providers.Vertex.APIKey,
CredentialsFile: cfg.Providers.Vertex.CredentialsFile,
ProjectID: cfg.Providers.Vertex.ProjectID,
Region: cfg.Providers.Vertex.Region,
DefaultModel: cfg.Providers.Vertex.Model,
}
prov, err := providers.NewVertexProviderWithTimeout(vcfg)
if err != nil {
slog.Warn("vertex: initialization failed", "error", err)
} else {
registry.Register(prov)
slog.Info("registered provider", "name", "vertex", "region", cfg.Providers.Vertex.Region, "project", cfg.Providers.Vertex.ProjectID)
}
}
// Claude CLI provider (subscription-based, no API key needed)
if cfg.Providers.ClaudeCLI.CLIPath != "" {
cliPath := cfg.Providers.ClaudeCLI.CLIPath
@@ -323,6 +344,30 @@ func registerProvidersFromDB(registry *providers.Registry, provStore store.Provi
slog.Info("registered provider from DB", "name", p.Name)
continue
}
// Vertex supports ADC (empty api_key) — handle before the generic key guard.
if p.ProviderType == store.ProviderVertex {
vsettings := store.ParseVertexProviderSettings(p.Settings)
if vsettings == nil {
slog.Warn("vertex: missing project_id/region in settings, skipping", "name", p.Name)
continue
}
vcfg := providers.VertexConfig{
Name: p.Name,
CredentialsJSON: p.APIKey,
ProjectID: vsettings.ProjectID,
Region: vsettings.Region,
DefaultModel: vsettings.Model,
APIBaseOverride: p.APIBase,
}
prov, err := providers.NewVertexProviderWithTimeout(vcfg)
if err != nil {
slog.Warn("vertex: init from DB failed", "name", p.Name, "error", err)
continue
}
registry.RegisterForTenant(p.TenantID, prov)
slog.Info("registered provider from DB", "name", p.Name, "type", "vertex", "region", vsettings.Region)
continue
}
if p.APIKey == "" {
continue
cmd/gateway_tools_wiring.go
+113
View File
@@ -1,14 +1,23 @@
package cmd
import (
"context"
"fmt"
"log/slog"
"os"
"path/filepath"
"time"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/edition"
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
"github.com/nextlevelbuilder/goclaw/internal/workstation"
"github.com/nextlevelbuilder/goclaw/internal/workstation/security"
)
// wireExtraTools registers cron, heartbeat, session, message tools and aliases
@@ -30,6 +39,7 @@ func wireExtraTools(
// DateTime tool (precise time for cron scheduling, memory timestamps, etc.)
toolsReg.Register(tools.NewDateTimeTool())
toolsReg.Register(tools.NewWaitTool())
// Cron tool (agent-facing)
toolsReg.Register(tools.NewCronTool(pgStores.Cron))
@@ -149,3 +159,106 @@ func wireExtraTools(
return heartbeatTool, hasMemory
}
// wireWorkstationTools registers workstation_exec and claude_remote tools (Standard edition only).
// Phase 6: wires the real AllowlistChecker permission check replacing the deny-all sentinel.
// Phase 7: wires the activity sink for exec audit logging.
//
// Security model (argv-exec, no sh -c):
// - C1 fix: cmd is the binary name (argv[0]), not a shell command string — no shell injection possible.
// - C2 fix: NFKC normalization applied before any check — collapses Unicode lookalikes.
// - Default-deny: AllowlistChecker rejects any cmd not in workstation's allowlist.
// - Rate limit: 30 exec/min per agent+workstation, 300/hr per workstation.
//
// Also subscribes to workstation update/delete events to keep BackendCache and
// AllowlistChecker cache consistent with the database.
func wireWorkstationTools(
pgStores *store.Stores,
toolsReg *tools.Registry,
domainBus eventbus.DomainEventBus,
) func() {
if edition.Current().Name != "standard" {
return func() {}
}
if pgStores.Workstations == nil || pgStores.WorkstationLinks == nil {
slog.Warn("workstation tools skipped: workstation stores not initialised")
return func() {}
}
backendCache := workstation.NewBackendCache(pgStores.Workstations, 10*time.Minute)
workstationExecTool := tools.NewWorkstationExecTool(
pgStores.Workstations,
pgStores.WorkstationLinks,
backendCache,
domainBus,
)
claudeRemoteTool := tools.NewClaudeRemoteTool(workstationExecTool)
// Phase 6: wire real permission checker (AllowlistChecker + rate limiter).
if pgStores.WorkstationPermissions != nil {
allowlistChecker := security.NewAllowlistChecker(pgStores.WorkstationPermissions, 30*time.Second)
rateLimiter := security.NewWorkstationRateLimiter()
workstationExecTool.SetPermCheck(func(ctx context.Context, ws *store.Workstation, cmd string, args []string, env map[string]string) error {
// Rate limit check first (cheap, no DB).
agentID := store.AgentIDFromContext(ctx).String()
if !rateLimiter.Allow(ws.TenantID, ws.ID, agentID) {
locale := store.LocaleFromContext(ctx)
return fmt.Errorf("%s", i18n.T(locale, i18n.MsgWorkstationRateLimit))
}
// Env blocklist check — rejects forbidden/sensitive env keys.
if err := allowlistChecker.CheckEnv(ctx, ws, env); err != nil {
return err
}
// Allowlist + input validation (NFKC normalize, NUL/CRLF, binary match).
return allowlistChecker.Check(ctx, ws, cmd, args)
})
slog.Info("workstation tools registered (Standard edition; Phase 6 AllowlistChecker active)")
// Invalidate allowlist cache on permission changes.
if domainBus != nil {
domainBus.Subscribe(eventbus.EventWorkstationPermChanged, func(_ context.Context, e eventbus.DomainEvent) error {
if id, err := uuid.Parse(e.SourceID); err == nil {
allowlistChecker.Invalidate(id)
slog.Debug("workstation allowlist cache invalidated", "workstation_id", id)
}
return nil
})
}
} else {
slog.Warn("workstation tools registered with deny-all: WorkstationPermissions store not initialised")
}
toolsReg.Register(workstationExecTool)
toolsReg.Register(claudeRemoteTool)
// Subscribe to workstation update/delete events to evict stale BackendCache entries.
if domainBus != nil {
domainBus.Subscribe(eventbus.EventWorkstationUpdated, func(_ context.Context, e eventbus.DomainEvent) error {
if id, err := uuid.Parse(e.SourceID); err == nil {
backendCache.Invalidate(id)
slog.Debug("workstation backend cache invalidated on update", "workstation_id", id)
}
return nil
})
domainBus.Subscribe(eventbus.EventWorkstationDeleted, func(_ context.Context, e eventbus.DomainEvent) error {
if id, err := uuid.Parse(e.SourceID); err == nil {
backendCache.Invalidate(id)
slog.Debug("workstation backend cache invalidated on delete", "workstation_id", id)
}
return nil
})
// Phase 7: wire activity audit sink (persists exec done events + nightly prune).
if pgStores.WorkstationActivity != nil {
stopSink := workstation.WireActivitySink(domainBus, pgStores.WorkstationActivity)
slog.Info("workstation activity audit sink registered")
return func() {
stopSink()
pgStores.WorkstationActivity.Stop()
}
}
}
return func() {}
}
cmd/pkg-helper/main.go
+186 -17
View File
@@ -1,3 +1,5 @@
//go:build !windows
// pkg-helper is a root-privileged helper that listens on a Unix socket
// and executes apk add/del commands on behalf of the non-root app process.
// It is started by docker-entrypoint.sh before dropping privileges.
@@ -15,6 +17,7 @@ import (
"path/filepath"
"regexp"
"strings"
"sync"
"syscall"
"time"
)
@@ -28,8 +31,22 @@ const goclawGID = 1000
// validPkgName allows alphanumeric, hyphens, underscores, dots, @, / (scoped npm).
// Rejects names starting with - to prevent argument injection.
// Used by install/uninstall for pip/npm cross-runtime compatibility (historical).
var validPkgName = regexp.MustCompile(`^[a-zA-Z0-9@][a-zA-Z0-9._+\-/@]*$`)
// validApkName enforces the stricter Alpine package name grammar applied
// only to the `upgrade` action. install/uninstall keep validPkgName for
// pip/npm cross-runtime compat (historical).
// Valid: curl, libstdc++, gtk+3.0, ca-certificates, py3-pip.
// Invalid: CURL (uppercase), @scope/pkg (@), curl/extra (/), -pkg (leading hyphen).
var validApkName = regexp.MustCompile(`^[a-z0-9][a-z0-9._+-]*$`)
// apkMutex serializes all apk CLI invocations within the helper process.
// Alpine apk uses a file lock at /var/lib/apk/db.lock; parallel calls would
// return "unable to lock database" with poor UX. Serializing in-process
// avoids the retry loop.
var apkMutex sync.Mutex
type request struct {
Action string `json:"action"`
Package string `json:"package"`
@@ -38,10 +55,12 @@ type request struct {
type response struct {
OK bool `json:"ok"`
Error string `json:"error,omitempty"`
Code string `json:"code,omitempty"`
Data string `json:"data,omitempty"`
}
func main() {
slog.Info("pkg-helper: starting", "socket", socketPath)
slog.Info("pkg-helper: starting", "socket", socketPath, "protocol", "v2")
// Remove stale socket.
os.Remove(socketPath)
@@ -96,7 +115,12 @@ func main() {
case sem <- struct{}{}:
go func(c net.Conn) {
defer func() { <-sem }()
c.SetDeadline(time.Now().Add(30 * time.Second)) //nolint:errcheck
// Safety ceiling: 10-minute deadline to evict dead clients.
// This is NOT a per-operation timeout — clients set conn.SetDeadline
// from ctx.Deadline() for that. This ceiling prevents maxConns=3
// semaphore starvation (DoS) if a client stops reading/writing.
// Renewed after each successful scanner.Scan() in handleConn.
c.SetDeadline(time.Now().Add(10 * time.Minute)) //nolint:errcheck
handleConn(c)
}(conn)
default:
@@ -109,13 +133,23 @@ func main() {
func handleConn(conn net.Conn) {
defer conn.Close()
// scanner.Buffer: 64KB initial / 1MB max.
// 1MB ceiling is a CONTRACT: any action returning >1MB of output must either
// raise this ceiling (both here and in the client) or split into multiple JSON
// lines. Violating this silently truncates at scanner boundary → helper_error.
scanner := bufio.NewScanner(conn)
scanner.Buffer(make([]byte, 64*1024), 1024*1024)
encoder := json.NewEncoder(conn)
for scanner.Scan() {
// Renew the 10-min safety deadline after each successfully received line.
// Rationale: a slow-mirror apk upgrade that took 9m59s to complete is
// legitimate; the next request should get a fresh 10 minutes.
conn.SetDeadline(time.Now().Add(10 * time.Minute)) //nolint:errcheck
var req request
if err := json.Unmarshal(scanner.Bytes(), &req); err != nil {
encoder.Encode(response{Error: "invalid json"}) //nolint:errcheck
encoder.Encode(response{Error: "invalid json", Code: "validation"}) //nolint:errcheck
continue
}
@@ -124,34 +158,68 @@ func handleConn(conn net.Conn) {
}
}
func handleRequest(req request) response {
pkg := req.Package
// validatePkg checks that pkg is non-empty and matches the given regex.
// Returns (true, zero) on success; (false, error response) on failure.
func validatePkg(pkg string, re *regexp.Regexp) (bool, response) {
if pkg == "" {
return response{Error: "package required"}
return false, response{Error: "package required", Code: "validation"}
}
if !validPkgName.MatchString(pkg) {
return response{Error: "invalid package name"}
if !re.MatchString(pkg) {
return false, response{Error: "invalid package name", Code: "validation"}
}
return true, response{}
}
func handleRequest(req request) response {
switch req.Action {
case "install":
return doInstall(pkg)
ok, errResp := validatePkg(req.Package, validPkgName)
if !ok {
return errResp
}
return doInstall(req.Package)
case "uninstall":
return doUninstall(pkg)
ok, errResp := validatePkg(req.Package, validPkgName)
if !ok {
return errResp
}
return doUninstall(req.Package)
case "upgrade":
// upgrade uses stricter validApkName (no @, no /, lowercase-only).
ok, errResp := validatePkg(req.Package, validApkName)
if !ok {
return errResp
}
return doUpgrade(req.Package)
case "update-index":
// Read-only action: no package argument expected.
if req.Package != "" {
return response{Error: "update-index takes no package", Code: "validation"}
}
return doUpdateIndex()
case "list-outdated":
// Read-only action: no package argument expected.
if req.Package != "" {
return response{Error: "list-outdated takes no package", Code: "validation"}
}
return doListOutdated()
default:
return response{Error: fmt.Sprintf("unknown action: %s", req.Action)}
return response{Error: fmt.Sprintf("unknown action: %s", req.Action), Code: "validation"}
}
}
func doInstall(pkg string) response {
apkMutex.Lock()
defer apkMutex.Unlock()
slog.Info("pkg-helper: installing", "package", pkg)
cmd := exec.Command("apk", "add", "--no-cache", pkg)
out, err := cmd.CombinedOutput()
if err != nil {
msg := fmt.Sprintf("%s: %v", strings.TrimSpace(string(out)), err)
slog.Error("pkg-helper: install failed", "package", pkg, "error", msg)
return response{Error: msg}
msg, code := classifyApkOutput(string(out), err)
slog.Error("pkg-helper: install failed", "package", pkg, "error", msg, "code", code)
return response{Error: msg, Code: code}
}
persistAdd(pkg)
@@ -160,14 +228,17 @@ func doInstall(pkg string) response {
}
func doUninstall(pkg string) response {
apkMutex.Lock()
defer apkMutex.Unlock()
slog.Info("pkg-helper: uninstalling", "package", pkg)
cmd := exec.Command("apk", "del", pkg)
out, err := cmd.CombinedOutput()
if err != nil {
msg := fmt.Sprintf("%s: %v", strings.TrimSpace(string(out)), err)
slog.Error("pkg-helper: uninstall failed", "package", pkg, "error", msg)
return response{Error: msg}
msg, code := classifyApkOutput(string(out), err)
slog.Error("pkg-helper: uninstall failed", "package", pkg, "error", msg, "code", code)
return response{Error: msg, Code: code}
}
persistRemove(pkg)
@@ -175,6 +246,104 @@ func doUninstall(pkg string) response {
return response{OK: true}
}
// doUpgrade runs `apk add -u <pkg>` to upgrade an existing package.
// Intentionally does NOT call persistAdd — upgrade does not change the installed set.
// The apk-packages file tracks what was explicitly installed, not version pinning.
func doUpgrade(pkg string) response {
apkMutex.Lock()
defer apkMutex.Unlock()
slog.Info("pkg-helper: upgrading", "package", pkg)
cmd := exec.Command("apk", "add", "-u", pkg)
out, err := cmd.CombinedOutput()
if err != nil {
msg, code := classifyApkOutput(string(out), err)
slog.Error("pkg-helper: upgrade failed", "package", pkg, "error", msg, "code", code)
return response{Error: msg, Code: code}
}
slog.Info("pkg-helper: upgraded", "package", pkg)
return response{OK: true}
}
// doUpdateIndex runs `apk update` to refresh the package index.
func doUpdateIndex() response {
apkMutex.Lock()
defer apkMutex.Unlock()
slog.Info("pkg-helper: updating index")
cmd := exec.Command("apk", "update")
out, err := cmd.CombinedOutput()
if err != nil {
msg, code := classifyApkOutput(string(out), err)
slog.Warn("pkg-helper: update-index failed", "error", msg, "code", code)
return response{Error: msg, Code: code}
}
slog.Info("pkg-helper: index updated")
return response{OK: true, Data: string(out)}
}
// doListOutdated runs `apk version -l '<'` to list packages with available upgrades.
// Returns stdout verbatim in the Data field.
func doListOutdated() response {
apkMutex.Lock()
defer apkMutex.Unlock()
cmd := exec.Command("apk", "version", "-l", "<")
out, err := cmd.CombinedOutput()
if err != nil {
msg, code := classifyApkOutput(string(out), err)
return response{Error: msg, Code: code}
}
return response{OK: true, Data: string(out)}
}
// classifyApkOutput inspects combined apk output + exit error and returns
// (truncated message, error code). This mirrors gateway-side ClassifyApkStderr
// but returns the code string directly (helper binary is separate from internal/skills).
//
// Code strings (authoritative for pkg-helper protocol):
// "locked", "permission", "disk_full", "not_found", "conflict", "network", "system_error".
//
// Note: "helper_unavailable" and "helper_error" are client-only codes; never emitted here.
func classifyApkOutput(out string, err error) (string, string) {
msg := strings.TrimSpace(out)
if msg == "" {
msg = err.Error()
}
if len(msg) > 500 {
msg = msg[:500] + "…"
}
lower := strings.ToLower(out)
switch {
case strings.Contains(out, "unable to lock"):
return msg, "locked"
case strings.Contains(out, "Permission denied"):
return msg, "permission"
case strings.Contains(out, "No space left on device"):
return msg, "disk_full"
case strings.Contains(out, "unsatisfiable constraints"):
if strings.Contains(out, "breaks: world") || strings.Contains(out, "required by") {
return msg, "conflict"
}
return msg, "not_found"
case strings.Contains(out, "breaks: world"):
return msg, "conflict"
case strings.Contains(lower, "network") ||
strings.Contains(out, "unable to fetch") ||
strings.Contains(out, "connection") ||
strings.Contains(out, "timed out") ||
strings.Contains(out, "hostname resolution failed"):
return msg, "network"
default:
return msg, "system_error"
}
}
// persistAdd appends a package name to the apk persist file (dedup check).
func persistAdd(pkg string) {
listFile := apkListFile()
cmd/pkg-helper/main_test.go
+363 -23
View File
@@ -1,7 +1,10 @@
//go:build !windows
package main
import (
"encoding/json"
"strings"
"testing"
)
@@ -119,24 +122,24 @@ func TestValidPkgName(t *testing.T) {
{"pkg_with_underscores", true},
{"pkg.with.dots", true},
// Invalid package names
{"-invalid", false}, // starts with hyphen
{"--flag", false}, // starts with hyphen
{"pkg name", false}, // contains space
{"pkg;cmd", false}, // contains semicolon
{"pkg|cmd", false}, // contains pipe
{"pkg&cmd", false}, // contains ampersand
{"pkg`cmd`", false}, // contains backtick
{"pkg$var", false}, // contains dollar sign
{"pkg<file", false}, // contains angle bracket
{"pkg>file", false}, // contains angle bracket
{"pkg'quote", false}, // contains quote
{"pkg\"quote", false}, // contains quote
{"pkg(paren)", false}, // contains parens
{"", false}, // empty
{" curl", false}, // starts with space
{"curl ", false}, // ends with space
{"--index-url=evil", false}, // flag pattern
{"-u", false}, // short flag
{"-invalid", false}, // starts with hyphen
{"--flag", false}, // starts with hyphen
{"pkg name", false}, // contains space
{"pkg;cmd", false}, // contains semicolon
{"pkg|cmd", false}, // contains pipe
{"pkg&cmd", false}, // contains ampersand
{"pkg`cmd`", false}, // contains backtick
{"pkg$var", false}, // contains dollar sign
{"pkg<file", false}, // contains angle bracket
{"pkg>file", false}, // contains angle bracket
{"pkg'quote", false}, // contains quote
{"pkg\"quote", false}, // contains quote
{"pkg(paren)", false}, // contains parens
{"", false}, // empty
{" curl", false}, // starts with space
{"curl ", false}, // ends with space
{"--index-url=evil", false}, // flag pattern
{"-u", false}, // short flag
}
for _, tt := range tests {
@@ -269,11 +272,11 @@ func unmarshalRequest(jsonStr string, req *request) error {
// TestResponse_JSON tests response struct JSON marshaling.
func TestResponse_JSON(t *testing.T) {
tests := []struct {
name string
resp response
wantOK bool
wantErr string
omitErr bool
name string
resp response
wantOK bool
wantErr string
omitErr bool
}{
{
name: "success response",
@@ -429,3 +432,340 @@ func TestHandleRequest_SuccessPath(t *testing.T) {
})
}
}
// ── v2 tests ─────────────────────────────────────────────────────────────────
// TestHandleRequest_UpgradeValidation verifies that the upgrade action uses
// the stricter validApkName regex (lowercase only, no @, no /).
func TestHandleRequest_UpgradeValidation(t *testing.T) {
// Valid names for upgrade (lowercase apk grammar)
valid := []string{
"curl",
"libstdc++",
"gtk+3.0",
"ca-certificates",
"py3-pip",
}
for _, pkg := range valid {
t.Run("valid/"+pkg, func(t *testing.T) {
resp := handleRequest(request{Action: "upgrade", Package: pkg})
// Must pass validation (may fail at apk exec stage — that's OK in unit test)
if contains(resp.Error, "package required") || contains(resp.Error, "invalid package name") {
t.Errorf("upgrade %q should pass validation, got: %q", pkg, resp.Error)
}
if resp.Code == "validation" {
t.Errorf("upgrade %q got validation code unexpectedly", pkg)
}
})
}
}
// TestHandleRequest_UpgradeInjectionPatterns verifies 5 injection patterns are rejected.
func TestHandleRequest_UpgradeInjectionPatterns(t *testing.T) {
injections := []string{
"-malicious", // leading hyphen
"pkg;evil", // semicolon
"pkg evil", // space
"@edge/curl", // @ prefix (legacy npm compat — rejected by validApkName)
"UPPERCASE_PKG", // uppercase rejected by validApkName
}
for _, pkg := range injections {
t.Run(pkg, func(t *testing.T) {
resp := handleRequest(request{Action: "upgrade", Package: pkg})
if resp.OK {
t.Errorf("upgrade %q should be rejected but got OK=true", pkg)
}
if resp.Code != "validation" {
t.Errorf("upgrade %q: want Code=validation, got %q (error=%q)", pkg, resp.Code, resp.Error)
}
})
}
}
// TestHandleRequest_UpgradeRejectsLegacySymbols verifies that pkg@edge (accepted
// by legacy validPkgName for install/uninstall) is REJECTED by upgrade action
// via the stricter validApkName.
func TestHandleRequest_UpgradeRejectsLegacySymbols(t *testing.T) {
legacySymbols := []string{
"pkg@edge", // @ accepted by validPkgName, rejected by validApkName
"@scope/pkg", // npm scoped — rejected by validApkName
}
for _, pkg := range legacySymbols {
t.Run(pkg, func(t *testing.T) {
// Confirm install/uninstall ACCEPTS it (legacy compat)
installResp := handleRequest(request{Action: "install", Package: pkg})
if contains(installResp.Error, "invalid package name") {
t.Errorf("install %q should pass validPkgName validation, got %q", pkg, installResp.Error)
}
// Confirm upgrade REJECTS it (strict apk grammar)
upgradeResp := handleRequest(request{Action: "upgrade", Package: pkg})
if upgradeResp.Code != "validation" {
t.Errorf("upgrade %q: want Code=validation, got Code=%q error=%q", pkg, upgradeResp.Code, upgradeResp.Error)
}
})
}
}
// TestHandleRequest_UpdateIndexRejectsPackage verifies update-index rejects non-empty package.
func TestHandleRequest_UpdateIndexRejectsPackage(t *testing.T) {
resp := handleRequest(request{Action: "update-index", Package: "curl"})
if resp.OK {
t.Error("update-index with package should not return OK=true")
}
if resp.Code != "validation" {
t.Errorf("want Code=validation, got %q", resp.Code)
}
if !contains(resp.Error, "update-index takes no package") {
t.Errorf("error = %q, want to contain 'update-index takes no package'", resp.Error)
}
}
// TestHandleRequest_ListOutdatedRejectsPackage verifies list-outdated rejects non-empty package.
func TestHandleRequest_ListOutdatedRejectsPackage(t *testing.T) {
resp := handleRequest(request{Action: "list-outdated", Package: "curl"})
if resp.OK {
t.Error("list-outdated with package should not return OK=true")
}
if resp.Code != "validation" {
t.Errorf("want Code=validation, got %q", resp.Code)
}
if !contains(resp.Error, "list-outdated takes no package") {
t.Errorf("error = %q, want to contain 'list-outdated takes no package'", resp.Error)
}
}
// TestHandleRequest_UpdateIndexNoPackage verifies update-index passes validation with empty package.
func TestHandleRequest_UpdateIndexNoPackage(t *testing.T) {
resp := handleRequest(request{Action: "update-index", Package: ""})
// Validation passes — will fail at apk exec in unit test env, but NOT with Code="validation"
if resp.Code == "validation" {
t.Errorf("update-index with empty package should pass validation, got Code=validation error=%q", resp.Error)
}
}
// TestHandleRequest_ListOutdatedNoPackage verifies list-outdated passes validation with empty package.
func TestHandleRequest_ListOutdatedNoPackage(t *testing.T) {
resp := handleRequest(request{Action: "list-outdated", Package: ""})
if resp.Code == "validation" {
t.Errorf("list-outdated with empty package should pass validation, got Code=validation error=%q", resp.Error)
}
}
// TestHandleRequest_InvalidActionReturnsValidationCode verifies unknown actions
// get Code="validation" in the v2 response.
func TestHandleRequest_InvalidActionReturnsValidationCode(t *testing.T) {
resp := handleRequest(request{Action: "nuke", Package: "curl"})
if resp.Code != "validation" {
t.Errorf("unknown action: want Code=validation, got %q", resp.Code)
}
if !contains(resp.Error, "unknown action") {
t.Errorf("error = %q, want to contain 'unknown action'", resp.Error)
}
}
// TestHandleRequest_InvalidJSONCodeValidation verifies malformed JSON sets Code="validation".
// We test via handleConn indirectly by confirming the inline code path.
func TestHandleRequest_InvalidJsonGetsValidationCode(t *testing.T) {
// This tests the inline json error path in handleConn — we verify the
// response struct used there has Code="validation".
errResp := response{Error: "invalid json", Code: "validation"}
if errResp.Code != "validation" {
t.Errorf("invalid json response Code = %q, want 'validation'", errResp.Code)
}
}
// TestClassifyApkOutput covers all 7 code branches.
func TestClassifyApkOutput(t *testing.T) {
fakeErr := &fakeError{"exit status 1"}
tests := []struct {
name string
out string
wantCode string
}{
{
name: "locked database",
out: "ERROR: unable to lock database: Permission denied",
wantCode: "locked",
},
{
name: "permission denied (not lock-related)",
out: "ERROR: Permission denied while writing",
wantCode: "permission",
},
{
name: "disk full",
out: "ERROR: No space left on device",
wantCode: "disk_full",
},
{
name: "not found (unsatisfiable)",
out: "ERROR: unsatisfiable constraints: nonexistent-pkg (missing)",
wantCode: "not_found",
},
{
name: "conflict (breaks world)",
out: "ERROR: unsatisfiable constraints: foo-1.0 breaks: world[foo=2.0]",
wantCode: "conflict",
},
{
name: "network error",
out: "ERROR: unable to fetch https://dl-cdn.alpinelinux.org/: connection refused",
wantCode: "network",
},
{
name: "system error (default)",
out: "ERROR: something completely unknown went wrong",
wantCode: "system_error",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, code := classifyApkOutput(tt.out, fakeErr)
if code != tt.wantCode {
t.Errorf("classifyApkOutput(%q) code = %q, want %q", tt.out, code, tt.wantCode)
}
})
}
}
// fakeError implements the error interface for testing classifyApkOutput.
type fakeError struct{ msg string }
func (e *fakeError) Error() string { return e.msg }
// TestClassifyApkOutput_EmptyOutputFallsBackToErrMsg verifies that when output
// is blank, the error message from err.Error() is used.
func TestClassifyApkOutput_EmptyOutputFallsBackToErrMsg(t *testing.T) {
msg, code := classifyApkOutput("", &fakeError{"apk: something failed"})
if msg != "apk: something failed" {
t.Errorf("msg = %q, want 'apk: something failed'", msg)
}
if code != "system_error" {
t.Errorf("code = %q, want 'system_error'", code)
}
}
// TestClassifyApkOutput_TruncatesLongOutput verifies messages >500 chars are truncated.
func TestClassifyApkOutput_TruncatesLongOutput(t *testing.T) {
longOut := strings.Repeat("x", 600)
msg, _ := classifyApkOutput(longOut, &fakeError{"err"})
if len([]rune(msg)) > 502 { // 500 + "…" (multi-byte)
t.Errorf("msg length = %d runes, want ≤502", len([]rune(msg)))
}
if !strings.HasSuffix(msg, "…") {
t.Error("truncated msg should end with ellipsis")
}
}
// TestResponseJSONShape verifies Code + Data fields survive marshal/unmarshal
// and that omitempty suppresses empty fields.
func TestResponseJSONShape(t *testing.T) {
t.Run("code and data present", func(t *testing.T) {
r := response{OK: false, Error: "x", Code: "conflict", Data: ""}
data, err := json.Marshal(r)
if err != nil {
t.Fatalf("marshal: %v", err)
}
s := string(data)
if !contains(s, `"code":"conflict"`) {
t.Errorf("json %q missing code field", s)
}
// Data is empty string — omitempty should suppress it.
if contains(s, `"data"`) {
t.Errorf("json %q should NOT contain data field when empty (omitempty)", s)
}
})
t.Run("data field present when non-empty", func(t *testing.T) {
r := response{OK: true, Data: "curl 7.88\n"}
data, err := json.Marshal(r)
if err != nil {
t.Fatalf("marshal: %v", err)
}
s := string(data)
if !contains(s, `"data"`) {
t.Errorf("json %q missing data field", s)
}
})
t.Run("omitempty suppresses error and code on OK response", func(t *testing.T) {
r := response{OK: true}
data, err := json.Marshal(r)
if err != nil {
t.Fatalf("marshal: %v", err)
}
s := string(data)
if contains(s, `"error"`) {
t.Errorf("json %q should NOT contain error field (omitempty)", s)
}
if contains(s, `"code"`) {
t.Errorf("json %q should NOT contain code field (omitempty)", s)
}
})
}
// TestValidApkName tests the strict apk package name validator.
func TestValidApkName(t *testing.T) {
tests := []struct {
name string
valid bool
}{
// Valid apk names
{"curl", true},
{"libstdc++", true},
{"gtk+3.0", true},
{"ca-certificates", true},
{"py3-pip", true},
{"0launch", true}, // starts with digit — valid per apk grammar
// Invalid: uppercase
{"CURL", false},
{"OpenSSL", false},
// Invalid: @ prefix (npm compat — rejected by validApkName)
{"@scope/pkg", false},
// Invalid: slash
{"alpine/curl", false},
// Invalid: leading hyphen
{"-pkg", false},
// Invalid: spaces/metacharacters
{"pkg name", false},
{"pkg;evil", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := validApkName.MatchString(tt.name)
if got != tt.valid {
t.Errorf("validApkName.MatchString(%q) = %v, want %v", tt.name, got, tt.valid)
}
})
}
}
// TestApkMutex_SerializesConcurrentUpgrades verifies that concurrent upgrade
// validation calls do not race on the response struct or the mutex itself.
// Note: actual apk execution is absent in unit tests; we exercise dispatch only.
func TestApkMutex_SerializesConcurrentUpgrades(t *testing.T) {
const goroutines = 10
results := make(chan response, goroutines)
for i := 0; i < goroutines; i++ {
go func() {
// All pass validation; execution fails (no apk binary) — that's OK.
results <- handleRequest(request{Action: "upgrade", Package: "curl"})
}()
}
for i := 0; i < goroutines; i++ {
resp := <-results
// Must NOT be a validation error — the package name is valid.
if resp.Code == "validation" {
t.Errorf("concurrent upgrade got unexpected validation error: %q", resp.Error)
}
}
}
cmd/pkg-helper/main_windows.go
+13
View File
@@ -0,0 +1,13 @@
//go:build windows
package main
import (
"fmt"
"os"
)
func main() {
fmt.Fprintln(os.Stderr, "pkg-helper is only supported on Unix-like systems")
os.Exit(1)
}
docker-compose.yml
+1 -1
View File
@@ -44,7 +44,7 @@ services:
- GOCLAW_HOST=0.0.0.0
- GOCLAW_PORT=18790
- GOCLAW_CONFIG=/app/data/config.json
- GOCLAW_GATEWAY_TOKEN=${GOCLAW_GATEWAY_TOKEN:-}
- GOCLAW_GATEWAY_TOKEN=${GOCLAW_GATEWAY_TOKEN:?run ./prepare-env.sh or set GOCLAW_GATEWAY_TOKEN}
- GOCLAW_ENCRYPTION_KEY=${GOCLAW_ENCRYPTION_KEY:-}
- GOCLAW_SKILLS_DIR=/app/data/skills
# Debug
docs/00-architecture-overview.md
+72
View File
@@ -551,6 +551,78 @@ Six distinct workspace scenarios:
---
## 12. Webhook Subsystem
External systems trigger agents or send channel messages via the webhook subsystem without using the gateway token (WebSocket/bearer) protocol.
### Components
| Component | Location | Role |
|-----------|----------|------|
| Admin CRUD handlers | `internal/http/webhooks_admin.go` | Create/list/get/patch/rotate/revoke webhook rows |
| Auth middleware | `internal/http/webhooks_auth.go` | Bearer + HMAC verification, localhost gate, kind check, rate limit, idempotency |
| LLM endpoint | `internal/http/webhooks_llm.go` | `POST /v1/webhooks/llm` — sync (30s) + async dispatch |
| Message endpoint | `internal/http/webhooks_message.go` | `POST /v1/webhooks/message` — channel delivery with media |
| Rate limiter | `internal/http/webhooks_ratelimit.go` | Per-webhook + per-tenant token bucket |
| Idempotency | `internal/http/webhooks_idempotency.go` | `Idempotency-Key` header cache (24h TTL) |
| Media fetch | `internal/http/webhooks_media_fetch.go` | SSRF-guarded HEAD probe + MIME validation |
| Callback worker | `internal/webhooks/worker.go` | Poll loop, claim, agent invoke, HMAC sign, HTTP POST, retry |
| Backoff | `internal/webhooks/backoff.go` | Exponential schedule `[30s, 2m, 10m, 1h, 6h]` with ±10% jitter |
| Signing | `internal/webhooks/sign.go` | `Sign(key, ts, body)` → `X-Webhook-Signature: t=...,v1=...` |
| Callback limiter | `internal/webhooks/limiter.go` | Per-tenant concurrency cap for outbound delivery goroutines |
| Store interfaces | `internal/store/` | `WebhookStore`, `WebhookCallStore` |
| PG store | `internal/store/pg/webhook_store.go`, `webhook_call_store.go` | Tenant-scoped SQL |
| SQLite store | `internal/store/sqlitestore/` | Lite edition support |
| Migrations | `migrations/` (PG), `internal/store/sqlitestore/schema.sql` (SQLite) | `webhooks` + `webhook_calls` tables |
### Inbound Flow
```
POST /v1/webhooks/llm or /v1/webhooks/message
→ WebhookAuthMiddleware
body cap → bearer/HMAC auth → localhost gate → kind check
→ rate limit (per-webhook + per-tenant) → idempotency → inject context
→ Handler (LLM or Message)
sync: agent.Run(30s timeout) → 200 with output
async: store WebhookCallData{status=queued} → 202 {call_id}
```
### Outbound Callback Flow (async only)
```
WebhookWorker.pollOneTenant()
→ calls.ClaimNext(lease_token CAS) → execute goroutine
→ invokeAgent (30s) → build callbackPayload
→ SSRF re-validate callback_url
→ HMAC sign body → POST to callback_url
→ 2xx: UpdateStatus(done, lease_token) | 4xx: failed | 5xx/net: retry with backoff | 429: Retry-After
```
**Lease Token Idempotency:** Each call row has a `lease_token` (UUID). Worker claims the row only if it can CAS the token. On success, worker updates status with the token as proof of ownership. Stale/slow receivers cannot accidentally overwrite a faster delivery attempt.
**Secret Encryption:** The raw webhook secret is encrypted at rest via AES-256-GCM using the `GOCLAW_ENCRYPTION_KEY` environment variable (same key as LLM provider credentials). Database leaks do not compromise HMAC material. See `docs/webhooks.md` § 14 for details.
### Security Log Events
| Event | Level | Trigger |
|-------|-------|---------|
| `security.webhook.auth_failed` | Warn | Invalid bearer / HMAC |
| `security.webhook.hmac_invalid` | Warn (via auth_failed) | HMAC mismatch |
| `security.webhook.body_too_large` | Warn | Body exceeds cap |
| `security.webhook.localhost_only_violation` | Warn | Non-loopback caller on restricted webhook |
| `security.webhook.kind_mismatch` | Warn | Caller path vs webhook kind mismatch |
| `security.webhook.rate_limited` | Warn | Per-webhook or per-tenant rate cap hit |
| `security.webhook.tenant_mismatch` | Warn | Agent UUID does not match webhook tenant |
| `security.webhook.tenant_leak_attempt` | Warn | Channel belongs to different tenant |
| `security.webhook.ssrf_blocked` | Warn | `media_url` SSRF rejection |
| `security.webhook.callback_ssrf_blocked` | Warn | `callback_url` SSRF rejection at delivery |
| `security.webhook.worker_panic` | Error | Delivery goroutine panic caught |
| `security.webhook.admin_denied` | Warn | Non-admin access to admin CRUD routes |
See `docs/webhooks.md` for the full integrator reference (auth, retries, HMAC examples).
---
## Cross-References
| Document | Content |
docs/02-providers.md
+14 -4
View File
@@ -44,6 +44,16 @@ All HTTP-based providers (Anthropic, OpenAI-compatible, Codex) use 300-second ti
---
## Agent Model Fallback
Agents can define `model_fallback` as an ordered list of backup provider/model pairs. The agent's configured `provider` and `model` are always the primary route; fallback candidates are tried in UI order when the primary route returns a classifiable provider failure such as rate limit, overload, timeout, auth/billing failure, model-not-found, or unknown transport failure. Context overflow is not treated as fallback because it needs compaction, not a different model.
Fallback is runtime-only and per agent. Explicit `ProviderOverride` or `ModelOverride` requests bypass the fallback wrapper so manual runs, heartbeats, or call sites that intentionally choose a model keep exact override behavior.
Streaming fallback is conservative: backup models are tried only if the stream fails before any content, thinking, or image chunk is emitted.
---
## 2. Supported Providers
### Six Core Provider Types
@@ -52,7 +62,7 @@ All HTTP-based providers (Anthropic, OpenAI-compatible, Codex) use 300-second ti
|----------|------|----------|---------------|
| **anthropic** | Native HTTP + SSE | API key required | `claude-sonnet-4-5-20250929` |
| **claude_cli** | stdio subprocess + MCP | Binary path (default: `claude`) | `sonnet` |
| **codex** | OAuth Responses API | OAuth token source | `gpt-5.3-codex` |
| **codex** | OAuth Responses API | OAuth token source | `gpt-5.5` |
| **acp** | JSON-RPC 2.0 subagents | Binary + workspace dir | `claude` |
| **dashscope** | OpenAI-compat wrapper | API key + custom models | `qwen3-max` |
| **openai** (+ 10+ variants) | OpenAI-compatible | API key + endpoint URL | Model-specific |
@@ -561,7 +571,7 @@ Claude CLI inherits thinking support from the underlying Claude model. Thinking
## 12. Codex Provider
The Codex provider integrates with OpenAI's ChatGPT Responses API (OAuth-based), enabling access to gpt-5.3-codex model through the chatgpt.com backend. Unlike standard OpenAI endpoints, Codex uses OAuth token refresh and a custom response format with "phase" markers.
The Codex provider integrates with OpenAI's ChatGPT Responses API (OAuth-based), defaulting to `gpt-5.5` through the chatgpt.com backend. Unlike standard OpenAI endpoints, Codex uses OAuth token refresh and a custom response format with "phase" markers.
### Configuration
@@ -572,7 +582,7 @@ tokenSource := &MyTokenSource{} // implements TokenSource interface
provider := NewCodexProvider("codex", tokenSource, "", "")
// or specify custom API base and model:
provider := NewCodexProvider("codex", tokenSource,
"https://chatgpt.com/backend-api", "gpt-5.3-codex")
"https://chatgpt.com/backend-api", "gpt-5.5")
```
### API Endpoint
@@ -591,7 +601,7 @@ Codex returns structured responses with phase markers:
```json
{
"id": "...",
"model": "gpt-5.3-codex",
"model": "gpt-5.5",
"choices": [{
"message": {
"role": "assistant",
docs/04-gateway-protocol.md
+5 -1
View File
@@ -112,7 +112,7 @@ flowchart LR
|------|--------------------|
| viewer | `agents.list`, `config.get`, `sessions.list`, `sessions.preview`, `health`, `status`, `providers.models`, `skills.list`, `skills.get`, `channels.list`, `channels.status`, `cron.list`, `cron.status`, `cron.runs`, `usage.get`, `usage.summary` |
| operator | All viewer methods plus: `chat.send`, `chat.abort`, `chat.history`, `chat.inject`, `sessions.delete`, `sessions.reset`, `sessions.patch`, `cron.create`, `cron.update`, `cron.delete`, `cron.toggle`, `cron.run`, `skills.update`, `send`, `exec.approval.list`, `exec.approval.approve`, `exec.approval.deny`, `device.pair.request`, `device.pair.list` |
| admin | All operator methods plus: `config.apply`, `config.patch`, `agents.create`, `agents.update`, `agents.delete`, `agents.files.*`, `teams.*`, `channels.toggle`, `device.pair.approve`, `device.pair.revoke` |
| admin | All operator methods plus: `config.apply`, `config.patch`, `config.permissions.*`, `agents.create`, `agents.update`, `agents.delete`, `agents.files.*`, `teams.*`, `channels.toggle`, `device.pair.approve`, `device.pair.revoke` |
---
@@ -194,6 +194,10 @@ flowchart TD
| `config.apply` | Replace entire configuration |
| `config.patch` | Partial configuration update |
| `config.schema` | Get configuration JSON schema |
| `config.permissions.list` | List agent config permission rules |
| `config.permissions.check` | Preview effective permission for an agent, scope, config type, and user |
| `config.permissions.grant` | Add or update an agent config permission rule |
| `config.permissions.revoke` | Remove an agent config permission rule |
### Skills
docs/06-store-data-model.md
+17
View File
@@ -68,6 +68,23 @@ The `Stores` struct is the top-level container holding all PostgreSQL-backed sto
---
## Agent Model Fallback Storage
Agent rows include `model_fallback`, stored as JSONB in PostgreSQL and TEXT JSON in SQLite. The config is per-agent and normalized before runtime use:
- `enabled`: whether fallback is active.
- `strategy`: currently `priority_order`.
- `candidates`: ordered backup provider/model pairs. The primary agent provider/model is not stored in this list.
- `max_attempts`: optional cap across primary plus fallback candidates.
- `cooldown_enabled`: temporarily skips recently failing routes when enabled.
Migration versions:
- PostgreSQL: `000065_agent_model_fallback`.
- SQLite: schema v33 to v34.
---
## 3. Session Caching
The session store uses an in-memory write-behind cache to minimize database I/O during the agent tool loop. All reads and writes happen in memory; data is flushed to the persistent backend only when `Save()` is called at the end of a run.
docs/09-security.md
+38
View File
@@ -456,6 +456,44 @@ When concurrency limits are hit, the error message is written for LLM reasoning:
---
## 13. Package Management Security
### pkg-helper privilege model (v1 / v2)
The `pkg-helper` sidecar is the only root-privileged component of the gateway.
| Boundary | Detail |
|----------|--------|
| Socket path | `/tmp/pkg.sock` |
| Permissions | 0600 — owner `root`, accessible only to `goclaw` uid 1000 |
| Gateway process | Runs as uid 1000 (goclaw); never calls `apk` directly |
| Helper process | Runs as root inside the container; started by `docker-entrypoint.sh` before privilege drop |
Package name validation is defense-in-depth at three layers:
1. HTTP handler (`ValidateApkPackageName` — strict `^[a-z0-9][a-z0-9._+-]*$` regex)
2. `ApkUpdateExecutor.Update()` — same validator before socket dial
3. pkg-helper itself — validates again server-side before exec
### pkg-helper v2 (Phase 2b)
- **Trust boundary unchanged from v1:** `/tmp/pkg.sock` 0600 owned by `root`,
group-readable by `goclaw`.
- **New actions** (`upgrade`, `update-index`, `list-outdated`) run under the same
root privilege as v1 `install`/`uninstall`. No privilege escalation; same exec
path, new action names.
- **`code` field** on error responses enables HTTP handlers to map errors to
appropriate 4xx/5xx statuses without stderr parsing — eliminates the string-grep
anti-pattern that risked misclassification.
- **apk invocation serialization** via process-wide `sync.Mutex` (`apkMutex`)
prevents TOCTOU races between concurrent `install` + `upgrade` operations on the
`/var/lib/apk/db` lock file.
- **No new network surface:** pkg-helper has no HTTP listener; it uses the same
Unix socket as v1. The socket path (`/tmp/pkg.sock`) is unchanged.
- **Stderr truncation:** helper stderr captured by the gateway is truncated to
500 chars (ANSI-stripped) before logging — prevents path leakage and PII in logs.
---
## File Reference
| Module | Path | Purpose |
docs/14-skills-runtime.md
+38 -3
View File
@@ -1,6 +1,6 @@
# 14 - Skills Runtime Environment
How skills access Python, Node.js, and system tools inside the Docker container. Covers image variants, pre-installed packages, runtime installation, and security constraints.
How skills access Python, Node.js, and system tools inside Docker containers and bare-metal gateway deployments. Covers image variants, pre-installed packages, runtime installation, and security constraints.
---
@@ -90,6 +90,24 @@ PATH=/app/data/.runtime/npm-global/bin:/app/data/.runtime/pip/bin:$PATH
2. **Node.js**: `npm install -g <package>` installs to `/app/data/.runtime/npm-global/`. `NODE_PATH` includes both system globals (`/usr/local/lib/node_modules`) and runtime globals.
3. **Persistence**: Packages installed at runtime persist across tool calls within the same container lifecycle (volume-backed).
### Bare-Metal Ubuntu/Debian
When the gateway runs directly on Ubuntu/Debian instead of inside the Alpine Docker image:
1. `pip:<name>` still runs `pip3 install --break-system-packages <name>`.
2. `npm:<name>` runs `npm install -g <name>` with a GoClaw-owned prefix at `{runtimeDir}/npm-global` instead of `/usr/lib/node_modules`.
3. Bare system package names use `sudo -n apt-get install -y --no-install-recommends <name>`.
4. Compatibility aliases: `pip3` installs `python3-pip`; `github-cli` installs `gh`.
5. Installed apt packages are recorded in `{runtimeDir}/system-packages.json` so the System Packages table can show the user-facing name (`github-cli`) while checking the real apt package (`gh`).
6. `/tmp/pkg.sock` is Docker/Alpine-only and is not required on bare-metal Ubuntu/Debian.
Default `{runtimeDir}` resolution:
1. `RUNTIME_DIR`, when set.
2. `GOCLAW_DATA_DIR/.runtime`, when `GOCLAW_DATA_DIR` is set.
3. `/var/lib/goclaw/data/.runtime` on bare-metal Linux.
4. `/app/data/.runtime` in Docker-style runtime.
### Agent Guidance
The system prompt and UI should treat runtime availability as variant-dependent:
@@ -199,10 +217,27 @@ github:owner/repo[@tag]
```
Admin-only, SHA256-verified, ELF-validated, with a release-picker UI. Binaries
land in `/app/data/.runtime/bin/` (on `$PATH`). See
land in `{runtimeDir}/bin/` (on `$PATH`). See
[`docs/packages-github.md`](./packages-github.md) for syntax, configuration,
security posture, and troubleshooting (especially musl/glibc compatibility).
### Update Flow (Phase 1: GitHub only)
GitHub binaries support proactive update checking via:
- UI summary bar on the Runtime & Packages page (badge + Refresh + Update All)
- `/v1/packages/updates*` endpoints (master-scope for writes)
- Atomic two-phase `.bak` swap with automatic rollback
- ETag-aware polling (304 = zero rate-limit cost)
- Pre-release handling via regex + `release.prerelease` + semver ordering
See [`docs/packages-github.md`](./packages-github.md) § "Updating Installed
Packages" for the full contract, troubleshooting, and runbook.
Pip/npm/apk update flows are **deferred to Phase 2** — the `UpdateChecker` /
`UpdateExecutor` interfaces in `internal/skills/update_registry.go` are
designed for interface-based extension without Phase 1 refactor.
---
## 8. Skill Search (v3)
@@ -241,7 +276,7 @@ exclude_deps: # filter false positives from auto-scan; ignored when deps: is
| Prefix | Effect | Example |
|--------|--------|---------|
| `pip:` | Python pip install | `pip:psycopg2-binary`, `pip:requests>=2.31` |
| `npm:` | Global npm install | `npm:typescript` |
| `npm:` | Global npm install under GoClaw runtime prefix | `npm:typescript`, `npm:@aiagentwiki/cli` |
| `github:` | GitHub Releases installer (admin) | `github:cli/cli@v2.40.0` |
| `system:` | apk package via pkg-helper | `system:ffmpeg` |
| (bare) | Treated as system binary | `pandoc` |
docs/18-http-api.md
+8 -8
View File
@@ -148,7 +148,7 @@ can override or inherit routing behavior while keeping the main `provider` field
```json
{
"provider": "openai-codex",
"model": "gpt-5.4",
"model": "gpt-5.5",
"chatgpt_oauth_routing": {
"override_mode": "custom",
"strategy": "round_robin"
@@ -214,7 +214,7 @@ Agents can now store capability-aware GPT-5/Codex reasoning intent under top-lev
```json
{
"provider": "openai-codex",
"model": "gpt-5.4",
"model": "gpt-5.5",
"reasoning_config": {
"override_mode": "inherit"
}
@@ -364,11 +364,11 @@ Example response:
{
"models": [
{
"id": "gpt-5.4",
"name": "GPT-5.4",
"id": "gpt-5.5",
"name": "GPT-5.5",
"reasoning": {
"levels": ["none", "low", "medium", "high", "xhigh"],
"default_effort": "none"
"default_effort": "medium"
}
},
{
@@ -978,7 +978,7 @@ Team activity and audit trail.
## 20. Secure CLI Credentials
CLI authentication credentials for secure command execution. Requires **admin role** (full gateway token or empty gateway token in dev/single-user mode).
CLI authentication credentials for secure command execution. Requires **admin role** (gateway token or empty-token local/dev fallback).
| Method | Path | Description |
|--------|------|-------------|
@@ -1014,7 +1014,7 @@ CLI authentication credentials for secure command execution. Requires **admin ro
## 21. Runtime & Packages Management
Manage system (apk), Python (pip), and Node (npm) package installation in the GoClaw runtime container. These endpoints do not inspect host-level runtimes. Requires authentication. When `GOCLAW_GATEWAY_TOKEN` is empty (dev/single-user mode), all users get admin role and can manage packages.
Manage system (apk), Python (pip), and Node (npm) package installation in the GoClaw runtime container. These endpoints do not inspect host-level runtimes. Requires authentication. Empty-token admin access is limited to loopback local development or explicit `GOCLAW_ALLOW_INSECURE_NO_AUTH=1`; external binds require `GOCLAW_GATEWAY_TOKEN`.
### List Installed Packages
@@ -1178,7 +1178,7 @@ Workspace file management.
| `GET` | `/v1/storage/files` | List files with depth limiting |
| `GET` | `/v1/storage/files/{path...}` | Read file (JSON or raw) |
| `POST` | `/v1/storage/files` | Upload file (admin) |
| `DELETE` | `/v1/storage/files/{path...}` | Delete file/directory |
| `DELETE` | `/v1/storage/files/{path...}` | Delete file/directory (admin) |
| `PUT` | `/v1/storage/move` | Move/rename file (admin) |
| `GET` | `/v1/storage/size` | Stream storage size (Server-Sent Events, cached 60 min) |
docs/20-api-keys-auth.md
+15 -4
View File
@@ -32,6 +32,15 @@ Or in WebSocket `connect`:
The gateway token is compared using **constant-time comparison** (`crypto/subtle.ConstantTimeCompare`) in both HTTP and WebSocket auth paths to prevent timing attacks. The comparison reveals no information about where the provided token first differs from the expected token.
Externally reachable deployments must configure a gateway token. If `gateway.token` / `GOCLAW_GATEWAY_TOKEN` is empty while the gateway binds to `0.0.0.0`, `::`, or a non-loopback address, startup fails before the health endpoint reports ready.
Empty-token compatibility is only for local development:
- bind `GOCLAW_HOST` to loopback (`127.0.0.1`, `localhost`, or `::1`), or
- set `GOCLAW_ALLOW_INSECURE_NO_AUTH=1` explicitly.
The explicit opt-in applies to both HTTP and WebSocket. Do not use it on shared hosts, Docker ports exposed outside the machine, or production deployments.
---
## 2. API Keys
@@ -100,7 +109,7 @@ GoClaw tries authentication methods in this priority order:
1. **Gateway token** (exact match via constant-time comparison) → `RoleAdmin` or `RoleOwner` for configured owner IDs
2. **API key** (SHA-256 hash lookup in `api_keys` table) → role from scopes
3. **Browser pairing** (sender ID must be paired with "browser" device type) → `RoleOperator` (HTTP only; requires `X-GoClaw-Sender-Id` header)
4. **No auth configured** (backward compatibility: if no gateway token is set) → full-access dev mode
4. **No auth configured and local/dev mode explicitly allowed** → full-access dev mode
5. **No valid auth found** → `401 Unauthorized`
### HTTP Request Flow
@@ -116,13 +125,15 @@ flowchart TD
G -->|Yes| H[Derive role from scopes]
G -->|No| I{Gateway token configured?}
I -->|Yes| J[401 Unauthorized]
I -->|No| K[Full-access backward compat]
I -->|No| K{Local/dev fallback allowed?}
K -->|No| J
K -->|Yes| O[Full-access backward compat]
C -->|Check paired device| L{Device paired?}
L -->|Yes| M[RoleOperator]
L -->|No| J
E --> N[Authenticate request]
H --> N
K --> N
O --> N
M --> N
```
@@ -164,7 +175,7 @@ On successful API key authentication, `last_used_at` is updated asynchronously (
### Backward Compatibility
If no gateway token is configured (`gateway.token` is empty in `config.json`), unauthenticated requests run in backward-compatibility full-access mode. This enables self-hosted deployments without strict authentication. Once a gateway token is configured, all requests must authenticate or use browser pairing.
If no gateway token is configured (`gateway.token` is empty in `config.json`), unauthenticated requests run in backward-compatibility full-access mode only for loopback local development or when `GOCLAW_ALLOW_INSECURE_NO_AUTH=1` is set. Once a gateway token is configured, all requests must authenticate or use browser pairing.
---
docs/23-ai-agent-permission-matrix.md
+71
View File
@@ -0,0 +1,71 @@
# AI Agent Permission Matrix
This matrix documents the effective authorization layers for agent actions across channels, groups, and workspaces.
## Permission Layers
| Layer | Scope | Enforced By | Notes |
|-------|-------|-------------|-------|
| Tenant RBAC | Dashboard, HTTP, WebSocket RPC | `internal/permissions` | Viewer/operator/admin/owner. Admin methods include `config.permissions.*`. |
| Agent ownership/share | Agent visibility and management | `store.AgentStore.CanAccess` | Controls which agents a dashboard user can manage. |
| Channel membership | Platform delivery | Channel adapter | Platform can still reject outbound delivery after GoClaw allows it. |
| Agent config permissions | Agent config mutations from chat | `agent_config_permissions` | Matches by `agent_id`, `scope`, `config_type`, `user_id`, including wildcard rows. |
| Workspace file boundary | Filesystem access | tool sandbox/boundary checks | Prevents path escape and unsupported writes. |
| Context file boundary | Agent identity/context files | `ContextFileInterceptor` | Routes protected files to store and requires group writer permission in group contexts. |
## Agent Config Permission Rows
| Field | Examples | Meaning |
|-------|----------|---------|
| `scope` | `agent`, `group:*`, `group:zalo:123`, `group:telegram:-100`, `*` | Where the grant applies. |
| `config_type` | `file_writer`, `heartbeat`, `cron`, `context_files`, `*` | What action family the grant covers. |
| `user_id` | `123456`, `zalo-user-id`, `*` | Who the grant covers. `*` grants every member in the selected scope. |
| `permission` | `allow`, `deny` | Effective decision. Deny can override broader allow. |
Effective precedence:
1. Individual deny.
2. Individual allow.
3. Scope/user wildcard deny.
4. Scope/user wildcard allow.
5. Default deny.
## Channel Matrix
| Channel Context | Read Agent Output | Send Reply | Write Workspace File | Write Protected Context File | Grant All Members |
|-----------------|-------------------|------------|----------------------|------------------------------|-------------------|
| Dashboard | RBAC controlled | N/A | Admin/operator path, then workspace boundary | Admin path, then context interceptor | Use Permissions tab |
| Direct message | Agent/session access | Channel adapter | Allowed by workspace boundary | Allowed by agent/context rules | Usually not needed |
| Telegram group | Group scope + sender ID | Channel adapter | Requires `file_writer` when group-gated | Requires `context_files` or `file_writer` and real sender | `scope=group:telegram:<chatId>`, `user_id=*` |
| Zalo group | Group scope + sender ID | Channel adapter, group thread metadata | Requires `file_writer` when group-gated | Requires `context_files` or `file_writer` and real sender | `scope=group:zalo:<chatId>`, `user_id=*` |
| Discord guild/channel | Guild scope + sender ID | Channel adapter | Requires `file_writer` when guild-gated | Requires `context_files` or `file_writer` and real sender | `scope=guild:<id>` or matching group scope, `user_id=*` |
| Scheduled/proactive run | System sender | Channel adapter | Deny for group-gated file writes unless elevated context | Deny for protected group context writes | Configure explicit rules or run from dashboard/admin context |
## Zalo Context Write Rule
Zalo group failures commonly happen when an agent writes `SOUL.md`, `IDENTITY.md`, `AGENTS.md`, `USER.md`, `USER_PREDEFINED.md`, or `CAPABILITIES.md` from a group session but the acting sender is missing. Protected context writes now use the group permission gate:
- `sender_id` must be a real platform user, not empty or synthetic.
- `user_id` must identify the group scope, for example `group:zalo:<chatId>`.
- The sender must match a `context_files` allow or legacy `file_writer` allow, including wildcard rows such as `user_id="*"`.
- Missing tenant context or permission-store errors fail closed.
## UX Contract
The Permissions tab should expose a full matrix editor:
| Control | Behavior |
|---------|----------|
| User/contact picker | Accepts explicit user IDs and contact search results. |
| All members button | Sets `user_id="*"` for the current rule. |
| Config type selector | Supports `file_writer`, `heartbeat`, `cron`, `context_files`, and `*`. |
| Scope selector | Supports known groups, `group:*`, `agent`, and `*`. |
| Check access | Calls `config.permissions.check` and shows the effective allow/deny decision before or after saving. |
## Security Notes
- Wildcard `user_id="*"` should be easy to grant but visually explicit because it expands access to every member in scope.
- Synthetic senders remain denied for group file/context writes. This avoids system turns inheriting permissions from no real user.
- Permission-store errors fail closed for group mutation boundaries.
- Backend validation rejects unknown config types and permissions before writing rules.
- Platform send permissions are still separate from GoClaw permissions; a channel adapter may reject delivery even when GoClaw allows the agent action.
docs/23-multi-tenant-architecture.md
+1 -1
View File
@@ -158,7 +158,7 @@ GoClaw determines the tenant from the credentials used to connect:
| **API key** (tenant-bound) | Auto from key's `tenant_id` | Normal SaaS integration |
| **API key** (system-level) + `X-GoClaw-Tenant-Id` | Header value (UUID or slug), while keeping the key's original role | Cross-tenant tools |
| **Browser pairing** | Master tenant by default, or a membership-validated tenant hint | Dashboard operators |
| **No credentials** | Master tenant | Dev/single-user mode |
| **No credentials** | Master tenant | Loopback local development or explicit `GOCLAW_ALLOW_INSECURE_NO_AUTH=1` only |
**Owner IDs:** Configured via `GOCLAW_OWNER_IDS` env var (comma-separated). Only owners get cross-tenant access with the gateway token. Default: `system`.
docs/codebase-summary.md
+50
View File
@@ -138,6 +138,56 @@ Native `image_generation` support in the Codex provider (`POST /codex/responses`
**Web UI:** Download filename resolver (`imageGenDownloadName`) in `ui/web/src/components/chat/media-gallery.tsx`. Image generation works automatically when the agent has the `create_image` tool — no user-facing toggle.
## Webhook Subsystem
External systems invoke agents or send channel messages via webhooks without gateway tokens.
### Components
| Path | Purpose |
|------|---------|
| `internal/http/webhooks_admin.go` | CRUD handlers (create, list, get, patch, rotate, revoke) |
| `internal/http/webhooks_auth.go` | Bearer + HMAC signature verification, IPAllowlist, tenant scope |
| `internal/http/webhooks_nonce.go` | Per-process HMAC replay cache (320s TTL) |
| `internal/http/webhooks_llm.go` | `POST /v1/webhooks/llm` endpoint (sync 30s / async) |
| `internal/http/webhooks_message.go` | `POST /v1/webhooks/message` endpoint (channel delivery) |
| `internal/http/webhooks_ratelimit.go` | Per-webhook + per-tenant rate limiting |
| `internal/http/webhooks_idempotency.go` | `Idempotency-Key` header dedup cache (24h TTL) |
| `internal/http/webhooks_media_fetch.go` | SSRF-guarded media URL fetch + MIME validation |
| `internal/webhooks/worker.go` | Async callback poller + delivery goroutines |
| `internal/webhooks/backoff.go` | Exponential retry schedule `[30s, 2m, 10m, 1h, 6h]` |
| `internal/webhooks/sign.go` | HMAC-SHA256 signing for outbound callbacks |
| `internal/webhooks/limiter.go` | Shared rate limiter for callback delivery |
| `internal/store/webhook_store.go` | `WebhookStore` interface + `WebhookCallStore` |
| `internal/store/pg/webhook_store.go` | PostgreSQL implementation (tenant-scoped) |
| `internal/store/sqlitestore/webhook_store.go` | SQLite implementation (Lite edition) |
| `migrations/` | PG migrations 000056–000058 (webhooks + lease token + encrypted secret) |
### Auth Flow
1. **Bearer auth**: Hash the token, lookup `secret_hash` globally (via `GetByHashUnscoped`) → return webhook + tenantID.
2. **HMAC auth**: Parse `X-Webhook-Id` header, lookup webhook globally → verify signature timestamp + nonce.
3. **Tenant inject**: Re-scope context with webhook's tenantID for all downstream calls.
4. **IP allowlist**: If non-empty, check request source IP (CIDR or exact) against list. Empty = allow all.
5. **Rate limit**: Check per-webhook + per-tenant buckets. Either rejects = 429.
### Idempotency & Lease Tokens
- **Inbound**: `Idempotency-Key` header dedup (24h cache). Same key + same body = cached response; same key + different body = 409 Conflict.
- **Outbound**: Each `webhook_calls` row has `lease_token` (UUID). Worker claims row with CAS. On update, token proves ownership — prevents stale receivers from overwriting.
### Secret Encryption
Raw webhook secret encrypted at rest via AES-256-GCM using `GOCLAW_ENCRYPTION_KEY` (same as LLM provider keys).
- Database: stores `encrypted_secret` column + `secret_hash` (for bearer lookups).
- DB compromise does not leak HMAC material.
- Clients receive plaintext secret once (create/rotate response) — must store securely.
### Audit Payload
All webhook calls logged with canonical `{"body_hash":"<sha256-hex>","meta":{...}}` shape in `webhook_calls.request_payload` (JSON).
Used by idempotency checker to detect body mismatches on replay.
---
## Key Conventions
docs/deployment-guide.md
+298
View File
@@ -0,0 +1,298 @@
# Deployment Guide
## Overview
Production target uses a hybrid deployment:
- GoClaw gateway runs as a bare-metal `systemd` service.
- PostgreSQL 18 + pgvector runs in Docker.
- Nginx reverse proxies public HTTP/HTTPS traffic to GoClaw on localhost.
- Codex CLI is installed on the host for future agent-controlled CLI work.
Current VPS shape:
| Item | Value |
|---|---|
| Host | Private deployment value |
| SSH | Private deployment value |
| Domain | Private deployment value |
| Gateway listen | `127.0.0.1:18790` |
| Public proxy | Nginx on `80/443` |
| Postgres | Docker container `goclaw-postgres` |
| Postgres bind | `127.0.0.1:5432` |
Keep concrete hostnames, IPs, SSH users, SSH ports, and credentials in private operator notes or environment variables outside this public repository.
For command examples below:
```bash
export GOCLAW_HOST=<server-ip-or-hostname>
export GOCLAW_SSH_USER=<ssh-user>
export GOCLAW_SSH_PORT=<ssh-port>
export GOCLAW_DOMAIN=<public-domain>
```
## Server Layout
| Path | Purpose |
|---|---|
| `/opt/goclaw/releases/<release>` | Immutable GoClaw release directory |
| `/opt/goclaw/current` | Symlink to active release |
| `/opt/goclaw/shared/docker-compose.postgres.yml` | Postgres compose file |
| `/opt/goclaw/shared/postgres.env` | Postgres env file |
| `/opt/goclaw/backups/` | Uploaded DB backup files |
| `/etc/goclaw/config.json` | Gateway config |
| `/etc/goclaw/goclaw.env` | Gateway env and secrets |
| `/etc/goclaw/r2-backup.env` | R2 backup env and secrets |
| `/var/lib/goclaw/data` | GoClaw persistent data |
| `/var/lib/goclaw/workspace` | Agent workspace |
| `/var/lib/goclaw/postgres` | Postgres Docker data |
| `/usr/local/bin/goclaw-deploy` | Release switch, upgrade, health-check, rollback |
| `/usr/local/bin/goclaw-issue-ssl` | Certbot wrapper for the deployment domain |
| `/usr/local/bin/goclaw-backup-r2` | Postgres dump, R2 upload, retention cleanup |
| `/usr/local/bin/goclaw-upgrade-release` | Download and deploy a GitHub Release tarball |
Secrets are stored only in server env files. Do not copy tokens or database passwords into repo docs.
## Runtime Services
Check status:
```bash
ssh -p "$GOCLAW_SSH_PORT" "$GOCLAW_SSH_USER@$GOCLAW_HOST"
systemctl status goclaw --no-pager
systemctl status nginx --no-pager
systemctl status goclaw-backup-r2.timer --no-pager
docker ps --filter name=goclaw-postgres
curl -fsS http://127.0.0.1:18790/health
```
Expected health:
```json
{"status":"ok","protocol":3}
```
## Initial Deployment Record
Completed on 2026-05-17:
- Installed Docker, Docker Compose v2, Nginx, Certbot, Node.js 22, Codex CLI.
- Added the operator workstation SSH public key to the deployment user.
- Installed Codex CLI. For agent-controlled `codex exec`, authenticate the Linux service user that runs GoClaw (`goclaw`), not only the SSH operator user.
- Restored the latest private PostgreSQL backup into Docker Postgres.
- Upgraded restored schema from `57` to `65`.
- Started the initial GoClaw release by `systemd`.
- Added automated database backup to a private Cloudflare R2 bucket.
- Verified local and Nginx health endpoints.
Current verification snapshot:
```text
goclaw_active=active
nginx_active=active
health_local={"status":"ok","protocol":3}
health_nginx={"status":"ok","protocol":3}
docker=goclaw-postgres pgvector/pgvector:pg18 healthy
schema=65
codex=codex-cli 0.130.0
```
## Codex CLI Auth For Agents
The gateway runs under the `goclaw` Linux user. Codex auth is stored under the
effective home directory, so an SSH-session login such as `codex login
--device-auth` under an operator user only writes that user's `~/.codex/auth.json`.
Agents invoking `codex` through the exec tool use the service user's home:
```text
/var/lib/goclaw/.codex/auth.json
```
Verify both contexts when debugging auth:
```bash
codex login status
sudo -u goclaw -H codex login status
```
If the operator user is logged in but `goclaw` is not, either run device auth as
the service user or copy the operator auth file with strict ownership:
```bash
sudo install -d -o goclaw -g goclaw -m 700 /var/lib/goclaw/.codex
sudo install -o goclaw -g goclaw -m 600 ~/.codex/auth.json /var/lib/goclaw/.codex/auth.json
sudo -u goclaw -H codex login status
sudo -u goclaw -H sh -lc 'mkdir -p /var/lib/goclaw/codex-smoke && cd /var/lib/goclaw/codex-smoke && codex exec --skip-git-repo-check --sandbox read-only "Reply with exactly: CODEX_AUTH_OK"'
```
Do not store Codex auth material in repository docs. Treat `auth.json` as a
credential.
## DNS And SSL
Cloudflare DNS record:
| Type | Name | Value | Proxy |
|---|---|---|---|
| `A` | Deployment subdomain | Deployment host IP | Proxied |
SSL was issued with Certbot for the deployment domain on 2026-05-17 and Certbot installed automatic renewal.
Verify HTTPS:
```bash
ssh -p "$GOCLAW_SSH_PORT" "$GOCLAW_SSH_USER@$GOCLAW_HOST"
curl -fsS "https://$GOCLAW_DOMAIN/health"
```
Re-issue manually if needed:
```bash
sudo /usr/local/bin/goclaw-issue-ssl
```
## Deploy A New Release
Preferred server-side upgrade flow:
```bash
sudo /usr/local/bin/goclaw-upgrade-release --dry-run latest
sudo /usr/local/bin/goclaw-upgrade-release latest
sudo /usr/local/bin/goclaw-upgrade-release v3.12.0
```
The script downloads the Linux amd64 GitHub Release tarball from `digitopvn/goclaw`, follows GitHub release redirects, verifies `CHECKSUMS.sha256`, extracts to `/opt/goclaw/releases/<tag>`, and calls `goclaw-deploy`.
The HTTP API still accepts only `tag`; it does not accept repo names or custom download URLs.
Remote API trigger is available in builds that include the gateway upgrade endpoint:
```bash
curl -fsS -X POST "https://$GOCLAW_DOMAIN/v1/system/gateway/upgrade" \
-H "Authorization: Bearer <gateway-token>" \
-H "X-GoClaw-Upgrade-Token: <upgrade-token>" \
-H "Content-Type: application/json" \
--data '{"tag":"latest"}'
```
Check status:
```bash
curl -fsS "https://$GOCLAW_DOMAIN/v1/system/gateway/upgrade/status" \
-H "Authorization: Bearer <gateway-token>" \
-H "X-GoClaw-Upgrade-Token: <upgrade-token>"
```
Keep upgrade tokens in server env files or secret managers. Do not put real tokens in docs.
The remote trigger endpoint fails closed unless `GOCLAW_UPGRADE_TRIGGER_TOKEN` is configured in the gateway environment.
Manual local-build fallback:
Build locally with embedded web UI:
```bash
cd ui/web
pnpm install --frozen-lockfile
pnpm build
cd ../..
rm -rf internal/webui/dist
mkdir -p internal/webui/dist
cp -r ui/web/dist/* internal/webui/dist/
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
go build -tags embedui \
-ldflags="-s -w -X github.com/nextlevelbuilder/goclaw/cmd.Version=<version>" \
-o dist/goclaw-linux-amd64 .
```
Upload:
```bash
release=<version-or-date>
ssh -p "$GOCLAW_SSH_PORT" "$GOCLAW_SSH_USER@$GOCLAW_HOST" "mkdir -p /opt/goclaw/releases/$release/migrations"
scp -P "$GOCLAW_SSH_PORT" dist/goclaw-linux-amd64 "$GOCLAW_SSH_USER@$GOCLAW_HOST:/opt/goclaw/releases/$release/goclaw"
scp -P "$GOCLAW_SSH_PORT" migrations/* "$GOCLAW_SSH_USER@$GOCLAW_HOST:/opt/goclaw/releases/$release/migrations/"
ssh -p "$GOCLAW_SSH_PORT" "$GOCLAW_SSH_USER@$GOCLAW_HOST" "chmod +x /opt/goclaw/releases/$release/goclaw && sudo /usr/local/bin/goclaw-deploy /opt/goclaw/releases/$release"
```
`goclaw-deploy` does:
1. Validate binary and migrations.
2. Switch `/opt/goclaw/current`.
3. Run `goclaw upgrade`.
4. Restart `goclaw`.
5. Poll `/health`.
6. Roll back symlink and restart if health fails.
## Backup And Restore
Automated backups:
| Item | Value |
|---|---|
| Timer | `goclaw-backup-r2.timer` |
| Schedule | Every 6 hours: `00:00`, `06:00`, `12:00`, `18:00` server time |
| Source | Docker Postgres container `goclaw-postgres` |
| Format | PostgreSQL custom dump, `pg_dump -Fc` |
| Local directory | `/opt/goclaw/backups/` |
| R2 bucket | Private deployment value |
| R2 prefix | Private deployment value |
| Retention | Keep latest 20 backups locally and in R2 |
Check timer and latest logs:
```bash
systemctl list-timers goclaw-backup-r2.timer --no-pager
journalctl -u goclaw-backup-r2.service -n 80 --no-pager
```
Run a manual backup:
```bash
sudo systemctl start goclaw-backup-r2.service
```
Create a database dump on server:
```bash
ts=$(date +%Y%m%d-%H%M%S)
docker exec goclaw-postgres pg_dump -U goclaw -Fc -d goclaw > /opt/goclaw/backups/goclaw-$ts.dump
```
Restore a dump:
```bash
systemctl stop goclaw
docker exec -i goclaw-postgres pg_restore -U goclaw -d goclaw --clean --if-exists --no-owner < /opt/goclaw/backups/<file>.dump
sudo /usr/local/bin/goclaw-deploy /opt/goclaw/current
```
## Operational Notes
- Gateway runs as Linux user `goclaw`.
- Host-control exceptions are deployment-specific and must be documented privately, not in this public runbook.
- Workspace restriction settings are deployment-specific and must be reviewed before enabling agent-controlled host operations.
- Postgres is bound only to localhost.
- UFW allows the private SSH port, `80/tcp`, and `443/tcp`.
- Reboot is recommended later because the VPS reports a pending kernel upgrade.
## Troubleshooting
Logs:
```bash
journalctl -u goclaw -n 200 --no-pager
docker logs goclaw-postgres --tail 100
tail -n 100 /var/log/nginx/error.log
```
Restart:
```bash
systemctl restart goclaw
docker compose --env-file /opt/goclaw/shared/postgres.env -f /opt/goclaw/shared/docker-compose.postgres.yml restart postgres
systemctl reload nginx
```
Unresolved questions: none.
docs/journals/260420-phase2b-apk-pkghelper-v2.md
+86
View File
@@ -0,0 +1,86 @@
# Phase 2b: Alpine APK Update Flow + pkg-helper v2 Protocol
**Date**: 2026-04-20 09:25
**Severity**: High (breaking protocol change)
**Component**: Packages update system (Alpine APK, pkg-helper IPC)
**Status**: Resolved
## Context
Completed Phase 2b of the packages-update feature: Alpine `apk` package update flow via privileged pkg-helper daemon. Commit `8fd0ba9f` merged to `feat/packages-update-phase2b-apk-pkghelper`. Feature gates at Standard/Full edition only (Lite unsupported). Stacked on Phase 2a (pip/npm) which was still unmerged at implementation time.
## Key Technical Decisions
**1. Non-root gateway → privileged helper for all apk ops**
- Initial scout assumed only write operations (`apk add/upgrade`) needed root. Audit revealed **both read and write are privileged**: `apk update` (fetch index) and `apk list --upgradable` (scan outdated) fail as uid 1000.
- Solution: route ALL apk CLI through helper IPC. Simpler than fine-grain permission escalation.
**2. pkg-helper v2 = breaking protocol atomic bump**
- Added `code`/`data` response fields (structured error classification + payload return).
- Expanded from 2 actions (install/uninstall) to 5: check_apk, check_pip, check_npm, exec_apk, exec_pip.
- No version field, no backward compatibility shim. Container/desktop upgrade boundary makes atomic rebuild cheap.
**3. Renewable 10-minute deadline instead of removing 30s**
- Red-team flagged: removing deadline lets maxConns=3 semaphore starvation cause indefinite hangs (DoS).
- Compromise: set before scanner loop, **renew per successful Scan**. Allows slow apk operations without exposing process-wide timeout bypass.
**4. Process-wide apkMutex inside helper**
- Alpine apk database is single-writer; `/var/lib/apk/db.lock` conflicts if gateway sent parallel requests.
- Helper serializes at apk boundary instead of retry loops in gateway.
**5. Executor acquires NO locks**
- `UpdateRegistry.Apply()` already holds `PackageLocker` (non-reentrant chan).
- Re-acquiring would deadlock. Documented in header; planner initially missed this pattern.
**6. Public SetAvailability() wrapper**
- Standard edition on non-Alpine host must emit `availability.apk=false` for UI (show "not applicable").
- Lite skips both registration and availability marker (key absent in response).
**7. Edition double-gate: compile-time + runtime**
- `edition.Current().SupportsApk && skills.IsAlpineRuntime()` — both must hold.
- Standard-Debian variants pass edition gate but fail `/etc/alpine-release` check.
- Lite on Alpine fails edition gate (even if runtime check passes).
**8. APK name regex allows `+` for libstdc++, gtk+3.0**
- Separate `validApkName` (stricter, lowercase-only) for apk-specific grammar.
- Keep historical `validPkgName` for install/uninstall (pip/npm cross-runtime compat).
## Red-Team Audit Catches (Pre-Code)
4 blocking issues surfaced in plan validation (trust-but-verify pattern, before Phase 1 started) — all resolved in phase files before implementation:
| Issue | Root Cause | Resolution |
|-------|-----------|-----------|
| C-1: Executor self-deadlock | Planner instructed to re-acquire PackageLocker | Removed re-acquire; document PackageLocker already held |
| C-2: No editor for availability map | SetAvailability() wrapper missing | Added public wrapper; wiring calls for Standard+non-Alpine |
| H-1: Deadline removal DoS | Naive removal of 30s cap | Renew-per-scan instead of unconditional remove |
| H-2: Zero-value edition silently disables | Default `bool` == false | Explicit `edition.SupportsApk = true` in Standard/Full presets |
## Outcomes
- **3,212 insertions**, 37 files modified
- **97/97 tests passing** (37.9s total, 0 race condition warnings)
- **Reviewer verdict**: APPROVE (0 critical/high/medium, 3 low cosmetic)
- Full stack: gateway wiring → checker → executor → helper protocol → frontend source pill
## Lessons
1. **Dockerfile verdict comes before code.** Permission model assumptions from package docs often diverge from actual runtime uid/gid. Inspect entrypoint and compare with runtime context (1000 vs 0).
2. **Breaking protocol changes are cheapest at atomic-rebuild boundaries.** Desktop/container upgrade boundaries make v1→v2 protocol jumps viable; avoid wire-compat shims unless two-operator rolling upgrade is in scope.
3. **Trust-but-verify Red-Team pattern works.** Scout → Planner → Red-Team audit (before token spend on implementation) caught structural deadlock and missing primitives. Prevented rework post-code.
4. **Renewable deadlines trade sophistication for safety.** Removing fixed timeout entirely opens DoS; renewing per-success-item lets slow operations complete while preventing starvation-based indefinite hangs.
5. **Edition double-gate (compile + runtime) beats runtime-only.** Catches mismatched environment early (Standard-Debian, Lite-Alpine) instead of silent availability glitches in production.
## Next Steps
- Phase 2b stacked on unmerged Phase 2a; await Phase 2a merge to main for CI/CD.
- Desktop .dmg release will auto-detect Alpine (via /etc/alpine-release sync.Once) and show apk sources in update UI.
- Standard edition: if deployed to non-Alpine, apk source shows "unavailable" (availability=false) instead of hidden.
**Unresolved**: none.
**Status**: DONE
docs/journals/260517-agent-provider-switch-json-null-shipped.md
+45
View File
@@ -0,0 +1,45 @@
# Agent Provider Switch JSON Null Fix
**Date**: 2026-05-17 21:52
**Severity**: High
**Component**: Agent persistence, provider/model switch flow, PG + SQLite stores
**Status**: Resolved
## What Happened
Saving an agent after switching provider or model failed because `chatgpt_oauth_routing:null` was preserved as a typed JSON nil/null and then written into config columns that are `NOT NULL`. The update path did not normalize empty or null-ish agent JSON config before hitting the store, so a routine UI action turned into a hard persistence failure.
## Assessment
This was a persistence-boundary bug. The UI action was routine, but the backend trusted a JSON shape that was not safe to persist. A provider/model switch could therefore write invalid config straight into the database. The fix belongs at the store boundary because HTTP, WebSocket, and future clients can all send equivalent null-ish JSON states.
## Technical Details
- Root trigger: `chatgpt_oauth_routing:null` survived as typed JSON null instead of being coerced away.
- Failure mode: store write hit `NOT NULL` JSON config columns in both PostgreSQL and SQLite.
- Fix: added store-level coercion for nil, empty, and JSON-null agent config updates in both PG and SQLite paths.
- Scope: persistence layer only; no product behavior change beyond accepting the update safely.
## What We Tried
1. Reproduced the failure in the runtime Docker stack to confirm it was not a UI-only issue.
2. Added focused store tests around nil/empty/JSON-null config updates.
3. Ran sqliteonly tests to catch the desktop path as well as PG.
4. Rebuilt Go targets and ran focused HTTP tests to verify the save flow end to end.
## Root Cause Analysis
The real mistake was letting typed JSON nulls flow through as if they were valid update payloads. The persistence layer assumed the caller had already normalized config state. That assumption was wrong for provider/model switch flows, where partial or empty config is normal and must be coerced before storage.
## Lessons Learned
1. Persisted JSON needs normalization at the store boundary, not just at the API boundary.
2. Nil and JSON null are not harmless variants when columns are `NOT NULL`.
3. PG and SQLite both need the same coercion logic or the bug just moves between editions.
## Next Steps
- Done: pushed branch `fix/agent-provider-switch-json-null`.
- Done: commits `324d9cf6` and `3da7ca51` included in the beta PR.
- Done: issue `#1148` linked for closure by the PR.
- Watch the next beta for any config-shape regressions around agent save flows.
docs/journals/260518-1741-beta-skill-grants-tenant-scope.md
+43
View File
@@ -0,0 +1,43 @@
# Beta Skill Grants Tenant-Scope Hardening
**Date**: 2026-05-18 17:41
**Severity**: High
**Component**: Skills grants, tenant isolation, skill management UI/API
**Status**: Resolved
## What Happened
This beta ship started from PR #14, which added per-agent skill manage grants so an authorized agent can edit/delete skills even when `owner_id` no longer matches its current actor identity. The review caught the ugly part: the grant path could mutate skill visibility or grant records across tenant boundaries, and the UI/API exposed `owner_id` in skill responses where clients did not need it. Issue digitopvn/goclaw#15 tracks the hardening fallout.
## The Brutal Truth
This was a permission feature shipped close to a tenant-isolation fault line. That is always where mistakes hurt most. The exhausting part is that the baseline feature was useful, but the first version trusted grant inputs too much. A beta is exactly where this should be caught, but it still feels bad because cross-tenant mutation risk is not a cosmetic bug.
## Technical Details
- PG now validates grant scope before insert/update in `internal/store/pg/skills_grants.go:19` and `verifySkillGrantScope` rejects mismatched agent tenants as `agent not found`.
- SQLite mirrors the same check in `internal/store/sqlitestore/skills_grants.go:25`.
- Cleanup migration `migrations/000067_skill_agent_grants_scope_cleanup.up.sql:1` deletes `skill_agent_grants` rows where `sag.tenant_id <> a.tenant_id` or non-system skill tenant differs.
- `internal/store/skill_store.go:19` hides `OwnerID` with `json:"-"`, stopping owner IDs from leaking through skill responses/UI.
- Changelog updated under `CHANGELOG.md`.
## What We Tried
- Kept PR #14's per-agent `can_manage` model because it directly solves agent-owned skill maintenance without pretending `owner_id` is stable agent identity.
- Rejected UI-only filtering because it would not protect HTTP, WebSocket, import, or future callers.
- Added store-layer verification in both PG and SQLite because that is the point every grant mutation must cross.
## Root Cause Analysis
We shipped the first feature around ownership semantics without enforcing the tenant relationship at the same boundary as the write. The fundamental mistake was treating authorization and scope as already resolved by callers. In a multi-tenant system, that assumption is how one tenant ends up mutating another tenant's visibility state.
## Lessons Learned
Grant writes must prove both sides of the relationship: the skill and the agent. Do not rely on caller discipline. Also, response structs should not expose security-sensitive ownership fields just because the database row has them.
## Next Steps
- Owner: maintainers. Monitor digitopvn/goclaw#15 through beta.
- Owner: release lead. Verify beta users receive migration `000067` and no stale cross-tenant rows survive upgrade.
- Owner: reviewers. Treat future grant/import/export changes as tenant-scope sensitive by default.
- Tests/build are passing for this ship; keep PG, SQLite, and Web UI build checks required before promoting beyond beta.
docs/journals/packages-update-phase1-github-260416.md
+158
View File
@@ -0,0 +1,158 @@
---
date: 2026-04-16
branch: feat/packages-update-flow
issue: nextlevelbuilder/goclaw#900
plan: plans/260415-1400-packages-update-flow/
status: shipped
severity: High
---
# Packages Update Flow Phase 1: What Went Wrong (And How We Caught It)
**Date**: 2026-04-16 16:35
**Issue**: [#900](https://github.com/nextlevelbuilder/goclaw/issues/900)
**Branch**: `feat/packages-update-flow`
**Completion**: 8 phases, 3.2k LOC, ship blockers identified and fixed before merge
## What We Built
Proactive update checker + atomic binary swap for GitHub-installed packages. ETag-based polling eliminates redundant GitHub API calls; SWR cache serves stale updates in background while refresh happens off-thread. Atomic `.bak`-rename swap ensures install↔update serialization and guaranteed rollback on failure. Interfaces ready for pip/npm/apk in Phase 2.
All 16 pre-flight hardening items from red-team review landed in code. Tests pass `-race`. Build works under both PostgreSQL and SQLite (`sqliteonly`) tags.
## What Went Wrong (And How We Caught It)
### CRIT-1: Double-Write HTTP Response on Invalid JSON Body
**Symptom**: Malformed JSON in `POST /v1/packages/apply-all` produces valid 200 response instead of 400 validation error.
**Root Cause**: `bindJSON(w, r, locale, &req)` writes its own 400 response on decode failure AND returns false. Handler ignored the bool (`_ = bindJSON(...)`), assumed empty body was valid, and executed with zero packages selected. Result: two HTTP status codes written, silent "apply everything" on corrupt input.
**Fix**: Read body into buffer first, check for empty explicitly (Content-Length 0 or io.EOF), skip JSON decode if empty, else call bindJSON with mandatory success. Three lines, compiles clean.
**Lesson**: Helpers that both write-and-return should never be called with `_ = ...`. Linter could catch this pattern (`"ignoring bool return from func that writes"`).
---
### CRIT-2: Lock-Key Divergence Between Installer and Update Executor
**Symptom**: Concurrent install of `cli/cli@vX` + update of `gh → vY` both execute without serialization, racing on manifest file.
**Root Cause**: Installer acquires lock on `parsed.Repo` ("cli/cli" → key `"github:cli"`). Executor acquires lock on the manifest `Name` via registry (`"github:gh"`). When `canonicalPackageName()` diverges, the "shared" PackageLocker doesn't actually serialize — they acquire different mutexes. The installer's internal `sync.Mutex` protects manifest writes, so data survives, but the invariant "one install/update per package at a time" is broken.
**Fix**: Both paths lock on the repo-portion of the spec, not the canonical name. Executor loads entry first, extracts repo, derives lock key from that. Both installer and executor now key by Repo — they serialize.
**Lesson**: "Shared locker" is a lie if the KEY is not shared. Document the key derivation rule explicitly. Unit test the rule: concurrent install+update on same package via both name and repo lookup should block.
---
### CRIT-3: Two-Phase Swap Rollback False-Alarms on Fresh Installs
**Symptom**: First-time package install, then update attempted → update fails mid-swap → rollback logs spurious `ENOENT` errors that wake ops, even though update failure was unrelated (e.g., download timeout).
**Root Cause**: Phase A (backup old binaries) skips entries where `os.Stat(dest)` returns ENOENT (fresh install). But Phase A still appends them to the rollback list. Phase B (move new binaries) then fails. Rollback code unconditionally calls `os.Rename(backup, dest)` for every entry — including ones where `backup` never existed, producing "rename ErrNotExist" logs. Alarm system treats these as rollback failures.
**Fix**: Add `hadBackup bool` flag to each swap target. Set true only after a real rename succeeds. Rollback skips where false. One extra bool per target, idempotent.
**Lesson**: Separate the "nothing to restore" branch from the "happy path." Don't let successful skips contaminate the rollback list. Think about the all-paths (nothing to backup, backup succeeds, backup fails, new fails, rollback succeeds, rollback fails) separately.
---
### HIGH-1: Lock Key Acquisition Spans Context Lifetime
**Symptom**: Acquire returns `(release, error)` but if ctx cancels after acquire, the release closure is never called, leak persists until goroutine exit.
**Root Cause**: `Acquire(ctx, source, name)` spawns a goroutine to monitor ctx cancelation. If ctx cancels before release() call, the release closure is never called by the caller. The monitor goroutine is never notified, lock never released.
**Fix**: `Acquire` uses `sync.Once` inside the release closure to make it idempotent; caller MUST `defer release()` immediately. Done. Tests verify defer pattern under context cancellation.
**Lesson**: Composable locks that return release closures should have single-call-only semantics. Document "must defer immediately." Test the defer+cancel path explicitly.
---
### HIGH-4: ETag Keyspace Collision Between Two Endpoints
**Symptom**: Pre-release user on `v1.0.0-rc.1` → GitHub releases stable `v1.0.0` → refresh checks both `/releases/latest` and `/releases?per_page=5` endpoints. ETag cache stored under one key ("lazygit"), so second endpoint 304 cache-hit masks the fact that latest changed.
**Root Cause**: `cache.GitHubETags["repo"]` used for both endpoints. Endpoints are independent resources with separate ETags. Storing both under one key means second endpoint's cache-hit shadows first endpoint's new data.
**Fix**: Two distinct keys: `cache.GitHubETags[repo]` and `cache.GitHubETags[repo + ":list"]`. Endpoints now have separate cache entries.
**Lesson**: Every GitHub endpoint is a resource with its own ETag. Do not alias. Document the key schema in the cache struct comment.
---
### MED: Pre-Release Transition Requires Semver Ordering
**Symptom**: User on `v1.0.0-rc.1`, stable `v1.0.0` released. Regex pre-release check (`(?i)-(alpha|beta|rc|...)`) flags current as pre-release, triggers dual-fetch. Naive string comparison would say `"v1.0.0-rc.1" < "v1.0.0"` is false (ASCII).
**Root Cause**: Pre-release handling was correct but the selector (`pickNewestRelease`) needed semver.Compare, not string inequality.
**Fix**: Import `golang.org/x/mod/semver`, use `semver.Compare(tag1, tag2)` for both-semver case. Falls back to string inequality for non-semver tags. Both functions return correct ordering.
**Lesson**: Check what production tools (Dependabot, Renovate) do before inventing ordering. Semver 2.0 has a clear spec; use it.
---
## Design Decisions That Paid Off
1. **Separate cache file** (not manifest bloat) — `/app/data/.runtime/updates-cache.json` is atomic tmp+rename, never touched by uninstall. Manifest path stays clean.
2. **Keyed lock shared between installer and update path** — Prevents install↔update race at logical boundary (locker key), not internal mutex. Extensible to pip/npm/apk in Phase 2 (all register checkers/executors with shared locker).
3. **SWR with `context.WithoutCancel`** — Background refresh on its own context, never blocks GET. Caller sees cache immediately + age metadata, decides staleness tolerance.
4. **ETag preservation verbatim** — Weak ETags kept with `W/` prefix, sent as-is in `If-None-Match`. No normalization, no parsing — delegates to GitHub's 304 logic.
5. **Rollback per-binary, not per-package** — Each binary swap is atomic; partial failure still leaves manifest consistent (we never write manifest until ALL binaries are moved). Forensic trace via `.failed-<ns>` dir.
6. **Red-team review pre-implementation** — 16 critical/high findings applied to plan before coding started. Post-implementation code review caught 3 more criticals. Total ~19 potential-production-bugs, caught before PR.
7. **Subagent parallelism worked** — Phase 4 (HTTP) + Phase 5 (events/i18n) + Phase 6 (frontend) ran in parallel; no file-ownership overlap. Combined context ~190K, fit well.
---
## Lessons for Phase 2
- Lock-key derivation is a contract. Document it in registry interface.
- Every HTTP endpoint has its own ETag; don't deduplicate.
- Helpers that write + return error should never be silently ignored; design API to prevent `_ = ...` pattern.
- Pre-release detection is simple; semver ordering is not — always use stdlib or battle-tested lib.
- Atomic swaps need explicit "nothing to swap" handling in rollback paths.
---
## Stats
| Metric | Count |
|--------|-------|
| Backend files created | 6 |
| Backend files modified | 8 |
| Frontend files created | 4 |
| Frontend files modified | 2 |
| Test files | 5 |
| Net LOC additions | 3,200 |
| Unit tests | 45+ |
| Integration tests | 1 |
| Benchmark tests | 2 |
| Build pass (PG + SQLite) | ✓ |
| `go vet` clean | ✓ |
| `-race` clean | ✓ |
| Code review status | APPROVE_WITH_CONDITIONS (3 critical fixes applied) |
| Red-team findings addressed | 16/16 |
---
## Open Questions / Tech Debt
1. **Multi-replica cache coherence**: Two gateway replicas share `/app/data/.runtime/updates-cache.json` — will race on `SaveUpdateCache`. Current single-process gateway is fine; document as invariant or add fd-lock.
2. **GitHubPackagesConfig.GitHubToken source**: Phase 1 stubs the field in JSON5. Phase 2 plan says env-only. Remove JSON field now or clarify intent.
3. **Secondary rate-limit ripple**: When `Check` aborts mid-sweep, partial Updates list is cached, so UI "forgets" already-known updates. Intended UX or should registry preserve prior Updates?
4. **Apply-all failure ordering**: Results preserve original slice order. Intentional? If so, document or implement stable ordering.
---
**Shipped**: 2026-04-16. All critical issues fixed. Ready for PR merge and Phase 2 (pip/npm/apk).
docs/journals/webhook-agent-triggering-260421-shipped.md
+66
View File
@@ -0,0 +1,66 @@
# Webhook Agent Triggering — Ship Complete
**Date**: 2026-04-21 23:59
**Severity**: Medium
**Component**: HTTP webhooks (inbound) + callback worker (outbound)
**Status**: Resolved
## What Happened
Shipped HTTP webhook API (POST /v1/webhooks/message + /v1/webhooks/llm) with callback delivery. Feature enables external systems to trigger agents synchronously or asynchronously, with outbound result delivery to a caller-specified callback URL. Dual-database (PostgreSQL Standard + SQLite Lite). 48 files, 9376 insertions. Nine sequential phases. Branch: feat/webhook-agent-triggering, commit 19e0c679.
## The Brutal Truth
Red-team review found the plan was unexecutable as written. Two fabricated API methods (`Router.Invoke`, `Manager.SendToChannel` media overload), three wrong file anchors, and four unspecified design decisions (media dispatch scope, callback idempotency, tenant concurrency, i18n ordering) meant that handing this to a teammate would have burned 4+ hours on false starts. After rework (2 hours of planner fixes), the plan was sound and execution was linear. The lesson: "trust-but-verify between planner and live code" is not optional — it catches real bugs before implementation wastes cycles.
## Technical Details
### Shipped contracts
- **POST /v1/webhooks/message**: Send text + media to channel. HMAC-SHA256 auth (X-GoClaw-Signature t=,v1=) + bearer token. Rate limit: per-webhook bucket (token refill 10/sec) + per-tenant global bucket (100/sec). Returns `{webhook_id, call_id}` immediately.
- **POST /v1/webhooks/llm**: Sync (wait for response, 30s timeout) or async (return call_id, deliver result to callback_url). Request body capped 1 MB; metadata capped 8 KB. HMAC + tenant-admin auth gate.
- Callback delivery: exponential backoff [30s, 2m, 10m, 1h, 6h] ±10% jitter, 5 attempts max. Outbound headers carry `X-Webhook-Delivery-Id` (stable across retries) for receiver-side dedupe. Claim uses FOR UPDATE SKIP LOCKED (PG) / BEGIN IMMEDIATE (SQLite).
### Critical decisions
1. **Callback idempotency:** `delivery_id` UUID on `webhook_calls.delivery_id` stays constant across retries. `attempts` counter incremented AFTER send completion (not before), so crash-restart never creates duplicates — receiver sees same delivery_id on retry. This invariant required reversing initial design ("increment on claim").
2. **Media dispatch:** Phase 05a added `channels.SendMediaToChannel()` because reused `SendToChannel(content string)` couldn't carry attachments. Grep found 8 adapters (telegram, discord, whatsapp, feishu, slack, zalo, pancake, facebook) already support `bus.OutboundMessage.Media` — not a new pattern. Phase 05b gates /message on `channels.IsMediaCapable(type)` with 501 fallback if unsupported.
3. **Tenant concurrency:** Per-tenant semaphore (sync.Map keyed by tenant_id → `*semaphore.Weighted`) with 5-minute TTL eviction. Prevents single tenant's callbacks from starving others. Non-blocking `TryAcquire` leaves row unclaimed on failure (no DB busy-loop); next 2s poll retries naturally.
4. **i18n front-loading:** All 19 keys × 3 catalogs (en/vi/zh) added upfront in phase 03, before any handler code. Prevents late-discovery "key not found" crashes. Phase 08 verifies the front-load.
## What We Tried
1. **Initial plan:** Router.Invoke entry point doesn't exist. Real pattern is `Router.Get(ctx, agentID) → Agent.Run(ctx, RunRequest)`, verified at `internal/agent/router.go:93` + `internal/agent/types.go:18`.
2. **Media dispatch design:** Planner assumed Manager.SendToChannel could carry attachments. Grep audit found it only took `content string`. Rework added dedicated `SendMediaToChannel(ctx, channelName, chatID, content, []bus.MediaAttachment)` method.
3. **Auth helpers location:** Plan cited `internal/http/auth.go` which doesn't define `requireTenantAdmin` or `requireMasterScope`. Grep found them at `internal/http/tenant_auth_helpers.go:22,71`.
4. **Edition gating:** Plan referenced nonexistent `edition.Current().Standard` and `.HasChannels()` methods. Rework added `AllowsChannels()` helper at `internal/edition/edition.go`.
## Root Cause Analysis
**Why the plan failed initial audit:** Planner reused API names from pattern prose without grepping live code. "Reuse Router.Invoke" sounded plausible for an entry point; the actual pattern is two-step (Get + Run). "Manager.SendToChannel carries media" was inferred from method naming, not from examining the struct definition. Edition gating was copy-paste from an older codebase pattern that didn't exist here.
**Why we caught it:** Red-team enforced CLAUDE Plan Verification Rule #3 ("no fabricated identifiers") and Rule #1 ("verify factual claims against code"). Spot-checks of 15+ claims against grep/line references surfaced every fabrication before implementation.
**Why rework was surgical, not rewrite:** The architecture (phases, concurrency model, auth gates) was sound. Only the API anchors and medium-sized design decisions needed fixing. Fixes were: (1) cite real entry points, (2) add one new channel method, (3) fix three file paths, (4) resolve four design questions. Execution then followed the reworked plan linearly, no surprises.
## Lessons Learned
1. **Trust-but-verify is load-bearing.** When a planner says "reuse X", don't delegate without a grep audit. Plausible-sounding APIs are the easiest to hallucinate. A 2-hour red-team pass caught what would have been 8+ hours of teammate confusion and rework.
2. **Crash-restart safety via immutable idempotency tokens is non-negotiable for async work.** Original design incremented attempts on claim; rework deferred it to post-send. This single decision eliminates the entire class of duplicate-delivery bugs on worker restart.
3. **Tenant isolation primitives (semaphores, TTL eviction, non-blocking acquire) scale better than ad-hoc limits.** Per-tenant semaphore with idle eviction is more complex than a simple global cap, but prevents the single-tenant-starves-others DoS and works at arbitrary scale.
4. **i18n keys as a blocker step, not a chore.** Front-loading all keys before handler code prevents runtime "key not found" crashes and makes phase dependencies explicit. Ordering matters more than scope.
5. **Anchoring API references is mechanical, not intuitive.** The plan correctly described what needed to be done (webhook auth, callback delivery, rate limiting) but cited wrong files/methods. Grep-by-symbol before writing. "Reuse X" must cite `file:line` and include a short signature snippet.
## Next Steps
1. Merge branch feat/webhook-agent-triggering → dev when CI green (currently in progress).
2. Monitor webhook_calls table cardinality and callback latency in first week post-deploy. Alert if p50 delivery time > 1 min (indicates tenant sem contention or stale reclaim pile-up).
3. v2 scope (deferred): /v1/webhooks/task (trigger workflows with task metadata), admin UI (web + desktop), callback secret rotation with grace window, observability dashboard for webhook metrics.
4. Document webhook integration pattern in `docs/webhooks.md` + provide client library examples (curl, Python, Go) for external systems.
docs/journals/webhook-fix-cycle-260421.md
+125
View File
@@ -0,0 +1,125 @@
# Webhook Fix Cycle — Quality Gates & Gap Closure
**Date**: 2026-04-21 02:15
**Severity**: High
**Component**: Webhook auth middleware, callback delivery state machine, encryption defaults
**Status**: Resolved
## What Happened
Post-ship code review (Stage 2 + Stage 3: quality + adversarial) on commit 19e0c679 surfaced 10 Critical/High findings across auth, concurrency, dual-database correctness, and security. Implemented 3-phase fix plan sequentially: (1) auth middleware ordering, (2) DB schema + driver compatibility, (3) encryption fail-fast + lease race closure. Re-audited fix diff, found 2 additional gaps. Final state: commit a83f4090, 54 files touched, all invariants passing.
## The Brutal Truth
This is the grind part of shipping features at scale. The original implementation was *architecturally sound* but *operationally fragile*. Ten issues surfaced not because the design was wrong, but because:
- **Stub stores hide real bugs.** Unit tests passed with fake stores; actual PG + SQLite layers rejected data or behaved differently.
- **Dual-DB testing is non-negotiable.** Developer tested on SQLite (local), which silently accepted data PG would reject. Production would have 100% failure.
- **Security-by-assumption kills in production.** Encryption code had a fail-open path: if `GOCLAW_ENCRYPTION_KEY` unset, new rows stored plaintext with zero operator signal.
- **Race conditions hide in "99.9% of the time works."** Slow receiver being re-claimed during send created duplicate delivery. CAS fixed it, but the gap existed because optimistic concurrency wasn't paranoid enough about lease semantics.
The frustrating part: all of this was *discoverable before ship* if we'd run Stage 2/3 reviews before commit. Instead, we shipped first, fixed second. Cost: 6 hours of emergency triage + review cycles. Won't repeat.
## Technical Details
### Issues fixed (10 + 2 re-audit gaps)
**K1 (Critical):** Auth middleware called store query BEFORE tenant context propagated. Flow: HTTP handler → auth middleware (queries all webhooks) → tenant context set. Fix: Moved context propagation upstream, updated middleware to accept tenant_id explicitly.
**K2 (Critical):** PG rejected `hexHash + jsonMeta` as 22P02 (bad JSONB format); SQLite BLOB silently accepted garbage. Root: developer tested schema on SQLite, passed CI (SQLite path). Fix: Added JSON validation layer + integration test enforcing both dbs reject invalid shapes.
**K3 (Critical — re-audit gap):** Reclaim handler returned 200 OK even when lease acquisition failed (non-blocking `TryAcquire`). Operator couldn't distinguish "reclaimed successfully" from "row still leased, will retry." Fix: Return 202 Accepted (idempotent ack) or 409 Conflict (retry backoff) explicitly.
**K4 (High):** Callback URL validation too lenient: `url.Parse()` only. Didn't reject `localhost`, `127.0.0.1`, or internal IPs. SSRF vector. Fix: Added explicit allowlist check against `config.CallbackIPAllowlist` + deny private ranges by default.
**K5 (High):** Slow receiver in flight when `reclaimStale` fired (90s window): row marked `stale`, reclaim reset to `queued`, but original delivery still in progress. Delivered twice. Fix: Added `lease_token` UUID column + WHERE lease_token matches on UpdateStatus. Only lease holder can transition state.
**K6 (High — re-audit gap):** `crypto.Encrypt("")` returns plaintext unchanged (side effect of AES-256-GCM no-op optimization). If `GOCLAW_ENCRYPTION_KEY` unset at startup, new webhook rows silently stored `encrypted_secret` as raw value. Operator had zero signal. HMAC still worked (doesn't care about value), so feature appeared functional. Fix: Skip-mount webhook routes during startup if key empty + throw 503 in admin handlers until key configured.
**K7 (High):** Tenant semaphore TTL eviction race: evicted semaphore while outstanding callbacks still lease-bound to it. New tenant gets fresh semaphore, old callbacks block on freed semaphore. Fix: Changed eviction to lazy-drop (mark invalid) instead of immediate removal; stale entries become no-op acquires.
**K8 (High):** i18n keys missing from `catalog_zh.go`. Feature shipped with English fallback silently replacing missing Chinese. Fix: Added all 19 keys to all 3 catalogs upfront (verified key-complete before code).
**K9 (Medium):** Rate limit bucket math wrong: intended 10/sec per webhook, implemented 10/sec per webhook + 100/sec global. Interaction unclear in docs. Fix: Clarified docs + added metric tags for bucket type to distinguish rates in observability.
**K10 (Medium):** SQLite schema migration `schema.go` missed `lease_token` column addition in incremental patch. Fresh desktop app would have column; upgraded lite app would not. Silent schema drift. Fix: Added patch explicitly + bumped SQLiteSchema version + added migration verify test.
**K3 re-audit:** Reclaim handler status codes.
**K6 re-audit:** Plaintext-fallback when key unset.
### Architecture
Original state machine (callback delivery):
```
PENDING → SENDING → (success) DELIVERED
↓ (timeout/error)
STALE → (reclaim fires) QUEUED → (retry) SENDING
```
Gap: if slow receiver still writing when reclaim fired, both paths advance row. K5 + lease_token fix closes it:
```
PENDING → [acquire lease_token] SENDING → (success) DELIVERED
↓ (timeout/error)
STALE → (reclaim fires, CAS on lease_token) QUEUED → SENDING
```
Only holder of lease_token can mutate state. Reclaim fails silently if lease held.
## What We Tried
1. **K1 fix v1:** Move auth to handler. Issue: auth middleware is reusable across endpoints. Better: context propagation moved outside middleware. Cost: 2 hours of middleware refactoring.
2. **K2 workaround (rejected):** "Make SQLite BLOB more strict." Issue: can't break SQLite's permissive typing. Real fix: validate before storing. Added JSON.Valid() gate at handler.
3. **K5 first attempt:** Increment attempts on claim instead of post-send. Issue: crash-restart during send would skip the increment, then resend on restart. Duplicate delivery again. Reverted; used immutable lease_token instead.
4. **K6 mitigation (rejected as insufficient):** Log warning if key unset. Issue: operator still ships plaintext to DB unknowingly. Real fix: refuse to start (no webhook routes mounted) until key configured.
5. **K7 race fix (rejected):** Atomic compare-and-swap on semaphore. Issue: Go's `sync.Map` doesn't support CAS. Changed to lazy eviction (write an invalid flag, read checks it).
## Root Cause Analysis
**Why K1-K10 existed:**
- **Stub stores.** Unit test suite used `&stubStore{}` that ignored all context. Auth middleware's actual behavior never tested against real store. Lesson: stubs prove wiring, not correctness.
- **Single-DB developer testing.** Feature developed on SQLite (dev environment). PG rejection of bad JSONB (K2) never hit. CI also runs on SQLite by default. Real schema validation only happens in integration tests on real databases.
- **Optimistic concurrency without paranoia.** Lease-based work queue is old pattern. Developer knew about `delivery_id` idempotency but missed lease semantics (who can mutate state?). Reclaim race (K5) is the *classic* slow-receiver bug in distributed systems.
- **Encryption-at-rest assumed secure.** Code comment said "encrypted secret stored." Developer didn't verify the encryption actually happened (fail-open path in crypto.Encrypt). Operator assumed safety because HMAC worked.
- **Dual-DB divergence unmonitored.** PG and SQLite migration systems are separate. K10 (missed SQLite patch) happened because no tooling checks "all PG migrations have SQLite equivalents." Manual discipline failed.
**Why we caught it:** Stage 2 + Stage 3 review on code (not running tests). Reviewers read auth flow, traced real store code, asked "what if key unset?" This is why adversarial review is load-bearing.
## Lessons Learned
1. **Stub stores prove wiring; integration tests prove correctness.** After this feature, all auth middleware routes require integration tests with real stores. Stubs are for unit tests only.
2. **Dual-DB testing is part of the build contract.** Add `make test-dual-db` that runs integration suite on both PG + SQLite variants. Gate CI on it. Single-database testing creates blind spots.
3. **Encryption-at-rest requires fail-fast, not fail-open.** Any "encrypted at rest" code path must refuse to boot in degraded mode. AES-256-GCM with unset key = app must not serve that handler. 503 or skip-mount, never silent plaintext.
4. **Optimistic concurrency needs explicit lease semantics.** Every work-queue (callback delivery, cron tasks, job workers) must define: who owns state? what operations require ownership? Write a state machine diagram before code. Lease token (UUID that changes on transition) is simpler than version numbers.
5. **Red-team review on fix diff catches implementer blind spots.** Original K1-K10 audit found issues. Adversarial re-audit on the fix diff found K3 + K6 gaps the implementer missed. 25% regression rate suggests re-audit is mandatory for fixes. Process: audit original → implement → red-team audit on diff → commit.
6. **Migration tooling debt surfaces in dual-DB systems.** Add a pre-commit hook that enumerates all migration names and verifies both PG + SQLite have entries (or explicitly exempted). Manual discipline isn't enough at 54-file scale.
## Next Steps
1. **Immediate (post-commit):** Merge a83f4090 → dev. Rerun all invariants + integration tests green. Monitor webhook_calls cardinality + callback latency on first week post-deploy.
2. **Short-term (this sprint):** Add `make test-dual-db` to CI. Require 100% pass on both PG + SQLite before merge. Enforce integration tests on all auth middleware routes.
3. **Medium-term (v2):** Implement migration-check pre-commit hook. Enumerate all migration identifiers at build time, verify dual-DB consistency. Document "lease semantics" pattern in `docs/patterns/optimistic-concurrency.md` for future work queues.
4. **Long-term:** Consider SQLite compile-time schema validation (build fails if schema.sql misses a migration). Evaluate telemetry for encryption key state (know when key unset). Both reduce operator surprise.
## Unresolved Questions
- Should K3 status code change (202 vs 409) be observable in dashboard? Currently metrics only. Consider adding webhook delivery status timeline to admin UI.
- Is per-webhook rate limit of 10/sec optimal? No production data yet to tune. Monitor p50/p95 delivery times first week, adjust if contention visible.
docs/packages-apk.md
+329
View File
@@ -0,0 +1,329 @@
# apk (Alpine Package Keeper) Updates (Phase 2b)
Extends the Phase 2a pip + npm update flow to Alpine Linux system packages.
GoClaw manages system packages via a privileged `pkg-helper` sidecar over a
Unix socket. This document covers how apk updates are detected, applied, and
what to do when things go wrong.
See also: [GitHub binary updates](./packages-github.md) · [pip + npm updates](./packages-pip-npm.md)
---
## 1. Overview
When the gateway runs inside an Alpine-based Docker image (`latest`, `full`,
`base`, `otel` variants) in **Standard edition**, `GET /v1/packages/updates`
includes system package updates alongside GitHub binaries, pip, and npm.
Two gates must both pass for apk to appear in the availability map:
1. **Runtime check:** `/etc/alpine-release` is present at startup. On Debian,
Ubuntu, or macOS desktop images, apk is silently omitted — no error, no update
results, `availability.apk = false`.
2. **Edition check:** `edition.Current().SupportsApk == true`. Standard
edition: always true. Lite desktop (macOS/Windows): always false — system
package management is not available outside containers.
Architecture note: the gateway process runs as `uid 1000` (goclaw) and never
calls `apk` directly. All apk operations are delegated to `/app/pkg-helper`
(root-owned), which listens on `/tmp/pkg.sock` (0600, accessible only to
goclaw). This keeps the main process unprivileged.
---
## 2. Command Matrix
Commands are executed inside `pkg-helper` (not by the gateway directly).
| Operation | Command inside helper | Timeout |
|---|---|---|
| Refresh index | `apk update` | 60 s |
| List outdated | `apk version -l '<'` | 30 s |
| Upgrade one package | `apk add -u <name>` | 5 min |
| Install new (dep install) | `apk add <name>` | 5 min |
| Remove | `apk del <name>` | 5 min |
The checker runs `apk update` + `apk version -l '<'` on every `Check()` call.
The executor runs `apk add -u <name>` on `POST /v1/packages/update`.
---
## 3. Behavior
### How the checker works
1. `GET /v1/packages/updates` triggers `ApkUpdateChecker.Check()`.
2. The checker sends an `update-index` action to pkg-helper (runs `apk update`
inside the container — refreshes the remote index from Alpine mirrors).
3. On success, it sends a `list-outdated` action (runs `apk version -l '<'`).
4. Output is parsed line-by-line. Each line has the form:
```
<name>-<installed_ver> < <available_ver>
```
The parser uses the rightmost `-<digit>` boundary to split name from version,
correctly handling names that contain hyphens (e.g. `py3-pip`, `ca-certificates`).
5. Malformed lines are skipped with a warning log; well-formed entries produce
`UpdateInfo` structs with `Source="apk"`.
6. Results are cached with the global `UpdatesCheckTTL` (default 1 hour).
The cache is invalidated on successful upgrade.
### Output parsing
`apk version -l '<'` format:
```
bash-5.2.21-r6 < 5.2.26-r0
py3-pip-22.0.4-r0 < 22.3-r0
ca-certificates-20230506-r0 < 20240226-r0
```
Name/version split uses the rightmost `hyphen-digit` boundary:
- `py3-pip-22.0.4-r0` → name=`py3-pip`, version=`22.0.4-r0`
- `ca-certificates-20230506-r0` → name=`ca-certificates`, version=`20230506-r0`
### How the executor works
`POST /v1/packages/update` with body `{"package": "apk:<name>"}`:
1. HTTP handler validates the package name (strict regex — no metacharacters).
2. `UpdateRegistry.Apply()` acquires a `PackageLocker` lock on `("apk", name)`.
3. `ApkUpdateExecutor.Update()` sends an `upgrade` action to pkg-helper.
4. pkg-helper acquires an in-process `sync.Mutex` (serializes all apk ops).
5. pkg-helper runs `apk add -u <name>`. On success, returns `{"ok":true}`.
6. On success, the cache entry for the package is removed; HTTP returns 200.
The per-source `PackageLocker` and the in-process `apkMutex` in pkg-helper
form a two-layer serialization guard:
- `PackageLocker`: prevents concurrent gateway-level operations on the same
`(source, name)` pair (e.g., dep install + update-apply racing).
- `apkMutex`: prevents concurrent apk database access from any code path
inside the helper process.
### pkg-helper v2 protocol
The helper uses a JSON line-oriented protocol over `/tmp/pkg.sock`:
**Request:**
```json
{"action": "upgrade", "package": "curl"}
```
**Success response:**
```json
{"ok": true, "data": ""}
```
**Error response:**
```json
{"ok": false, "error": "ERROR: unable to select packages", "code": "not_found"}
```
New v2 fields compared to v1:
- `code` — typed error classification (see Error Classes section)
- `data` — opaque payload for `list-outdated` results
- New actions: `upgrade`, `update-index`, `list-outdated`
v1 callers that omit `code` on error responses receive `system_error` by default
in the client — backward-compat for split deployments where helper is not yet
rebuilt. However, new actions (`upgrade`, `update-index`, `list-outdated`) return
`unknown action` on a v1 helper — feature is degraded, not crashed.
---
## 4. Pre-Release Handling
**Not applicable.** Alpine repositories do not distinguish stable vs pre-release
in the `apk version` output. `apk version -l '<'` lists all packages where the
installed version is older than the repository version. There is no pre-release
channel concept in the Alpine package ecosystem.
The apk checker always reports available upgrades without pre-release filtering.
---
## 5. Availability — Edition × Runtime Truth Table
| Edition | Runtime | `availability.apk` | apk checker registered? |
|---|---|---|---|
| Standard | Alpine (`/etc/alpine-release` present) | `true` | Yes |
| Standard | Debian / Ubuntu | `false` | No (runtime gate) |
| Standard | macOS (dev / testing) | `false` | No (runtime gate) |
| Lite (desktop) | Any | `false` | No (edition gate) |
When `availability.apk = false`:
- `GET /v1/packages/updates` response includes `"availability": {"apk": false}`.
- The frontend hides the apk source from the filter bar.
- `POST /v1/packages/update` with `apk:<name>` returns 503 (source not registered)
or 409 (Lite edition gate — source never wired).
The runtime check (`/etc/alpine-release` stat) is performed once at checker
initialization and cached. It does not re-probe on subsequent calls.
---
## 6. Error Classes
Sentinel errors are defined in `internal/skills/pkg_update_helpers.go`.
The `code` field in pkg-helper responses maps to these sentinels.
| Sentinel | code value | Trigger |
|---|---|---|
| `ErrInvalidApkPackageName` | `validation` | Package name fails regex (metacharacter, uppercase, etc.) |
| `ErrUpdateApkNotFound` | `not_found` | `apk add -u <name>` reports "unable to select" |
| `ErrUpdateApkConflict` | `conflict` or `constraint` | Dependency conflict / unsatisfiable constraints |
| `ErrUpdateApkLocked` | `locked` | `/var/lib/apk/db.lock` held by another process |
| `ErrUpdateApkNetwork` | `network` | Mirror fetch timeout, DNS failure |
| `ErrUpdateApkPermission` | `permission` | Write permission denied in `/var/lib/apk` |
| `ErrUpdateApkDiskFull` | `disk_full` | No space left on `/var/cache/apk` or `/` |
| `ErrUpdateApkHelperUnavail` | `helper_unavailable` | Socket dial failure (helper not running) |
Unclassified errors (`code=""` or `system_error`) fall back to `ClassifyApkStderr`
pattern matching, then to a generic wrapped error with truncated stderr (≤ 500 chars,
ANSI-stripped before logging).
HTTP status mapping (via `packages_updates.go`):
| Sentinel | HTTP status |
|---|---|
| `ErrInvalidApkPackageName` | 400 Bad Request |
| `ErrUpdateApkNotFound` | 404 Not Found |
| `ErrUpdateApkConflict` | 409 Conflict |
| `ErrUpdateApkLocked` | 409 Conflict |
| `ErrUpdateApkNetwork` | 502 Bad Gateway |
| `ErrUpdateApkPermission` | 500 Internal Server Error |
| `ErrUpdateApkDiskFull` | 500 Internal Server Error |
| `ErrUpdateApkHelperUnavail` | 503 Service Unavailable |
---
## 7. Runbook
### "pkg-helper unavailable" (503)
`/app/pkg-helper` is not running, or `/tmp/pkg.sock` does not exist.
For Docker Alpine deployments this is an error. For bare-metal Ubuntu/Debian
deployments, `/tmp/pkg.sock` is expected to be absent; package install should
use the apt path instead of apk/pkg-helper.
1. Check container logs: `docker logs <container> 2>&1 | grep pkg-helper`
2. Verify the binary exists: `docker exec <container> ls -la /app/pkg-helper`
3. If missing, the Docker image was NOT rebuilt after the pkg-helper v2 upgrade.
Pull the new image and recreate the container.
4. If the binary exists but the socket is missing, check that the container
entrypoint starts the helper before the gateway: `ENTRYPOINT ["/app/entrypoint.sh"]`.
### Bare-metal Ubuntu/Debian package table
On bare-metal Ubuntu/Debian, system package install does not write the Alpine
`apk-packages` persist file. GoClaw records successful apt installs in
`{runtimeDir}/system-packages.json` and lists versions via `dpkg-query`.
Alias examples:
- Installing `pip3` records display name `pip3`, apt package `python3-pip`.
- Installing `github-cli` records display name `github-cli`, apt package `gh`.
The System Packages table should show the display name users installed, not the
underlying Debian package alias.
### Bare-metal npm global prefix
On bare-metal Ubuntu/Debian, Node packages installed from the Packages page use
`{runtimeDir}/npm-global` as `NPM_CONFIG_PREFIX`. This avoids writing to
`/usr/lib/node_modules`, which is root-owned on standard Ubuntu installs.
Logging: the gateway emits `slog.Info("package.update.apk.unavailable")` when
the helper socket is unreachable. Grep for this key to confirm the symptom.
### "Package database is locked" (409)
`/var/lib/apk/db.lock` is held by another apk process.
1. Wait ~10 seconds and retry — an in-progress `apk add` from the dep-installer
may still be running (the apkMutex serializes gateway operations, but manual
`docker exec apk add` from outside the gateway bypasses it).
2. If the lock persists: `docker exec <container> ls -la /var/lib/apk/db.lock`
— if the owning PID is dead, the lock is stale. Restart the container.
3. Do NOT run `rm /var/lib/apk/db.lock` manually — apk may be mid-write.
Logging: `slog.Warn("package.update.apk.outcome", "code", "locked")`.
### "Disk full" (500)
`/var/cache/apk` or `/` is out of space.
1. Check disk: `docker exec <container> df -h /`
2. Clean cache: `docker exec <container> apk cache clean`
3. Expand the container volume or prune unused images on the host.
### "Dependency conflict" (409)
`apk` cannot resolve dependencies for the requested upgrade.
1. SSH into the container: `docker exec -it <container> sh`
2. Run manually: `apk add -u <name> --simulate` to see the conflict details.
3. Resolution typically requires upgrading a conflicting package first, or
accepting cascade upgrades. The GoClaw UI warns about cascade risk for
system packages.
4. If unresolvable, the package must be pinned via Dockerfile `RUN apk add`.
### Debugging helper protocol issues
The helper logs all actions to stderr (`docker logs <container>`). To trace
a specific action:
```bash
# Manual socket test (requires jq on PATH):
echo '{"action":"list-outdated","package":""}' | \
nc -U /tmp/pkg.sock | jq .
```
Expected response shape:
```json
{"ok": true, "data": "bash-5.2.21-r6 < 5.2.26-r0\n"}
```
---
## 8. Minimum Versions
| Component | Minimum | Notes |
|---|---|---|
| Alpine Linux | 3.19 | `apk version -l '<'` output format stable since 3.12; 3.19 tested |
| apk-tools | 2.14 | Bundled with Alpine 3.19+; older versions may have different `version -l` output |
| pkg-helper | v2 (Phase 2b) | v1 helpers lack `upgrade` / `update-index` / `list-outdated` actions |
| Docker image | Phase 2b build | Image must be rebuilt to include the new pkg-helper binary |
---
## 9. Fixture Regeneration
Test fixtures for the apk parser live in `internal/skills/testdata/`. When the
Alpine version is upgraded and `apk version -l '<'` output format changes:
```bash
# Capture live output from a running container:
docker exec <container> apk update && \
docker exec <container> apk version -l '<' \
> internal/skills/testdata/apk_outdated_alpine319.txt
# Verify the parser handles the new format:
go test -run TestParseApkOutdated ./internal/skills/...
# Update test cases in apk_update_checker_test.go to reference the new fixture
# and expected name/version values.
```
Fixture files are named with the Alpine version (`alpine319`) so drift between
CI environments is detectable by `git diff`.
### Updating pkg-helper v2 protocol tests
If the helper wire format changes (new fields, action names):
1. Update `apk_helper_call_test.go` — `servePkgHelper` / `dialHelper` helpers.
2. Update `apk_update_checker_test.go` and `apk_update_executor_test.go` —
canned response maps.
3. Update `cmd/pkg-helper/main_test.go` — v2 protocol action dispatch tests.
4. Run: `go test ./internal/skills/... ./cmd/pkg-helper/...` to verify.
docs/packages-github.md
+104 -3
View File
@@ -24,7 +24,7 @@ Examples:
4. Verifies SHA256 if the publisher ships `checksums.txt` / `SHA256SUMS`
5. Validates ELF magic bytes + 64-bit class + machine matches runtime arch
6. Extracts archive safely (tar.gz / zip / raw binary) with path-traversal + zip-bomb guards
7. Installs to `/app/data/.runtime/bin/` (prepended to `$PATH`)
7. Installs to `{runtimeDir}/bin/` (prepended to `$PATH`)
8. Persists a manifest for later listing + uninstall
## Usage
@@ -69,7 +69,7 @@ token in `config.json`.
| `GOCLAW_PACKAGES_GITHUB_TOKEN` | `""` | Optional PAT: rate 60/hr → 5000/hr + private repo access |
| `GOCLAW_PACKAGES_MAX_ASSET_SIZE_MB` | `200` | Applies to both download cap and 2× uncompressed cap |
| `GOCLAW_PACKAGES_GITHUB_ALLOWED_ORGS` | `""` | Comma-separated allowlist (empty = all orgs allowed) |
| `GOCLAW_PACKAGES_GITHUB_BIN_DIR` | `/app/data/.runtime/bin` | Where extracted binaries land |
| `GOCLAW_PACKAGES_GITHUB_BIN_DIR` | `{runtimeDir}/bin` | Where extracted binaries land |
| `GOCLAW_PACKAGES_GITHUB_MANIFEST` | `{bin_dir}/../github-packages.json` | Manifest path |
Token scopes:
@@ -133,13 +133,114 @@ the release. Do not force-install; report upstream.
## Limitations (Phase 1)
- Linux-only (Lite/Desktop editions not yet supported)
- Docker edition only (runtime dir `/app/data/.runtime/bin`)
- Docker and bare-metal gateway editions (default runtime dir resolves to `/app/data/.runtime/bin` in Docker or `/var/lib/goclaw/data/.runtime/bin` on bare-metal Linux)
- Installs all top-level executables in an archive (no interactive picker if
archive contains multiple binaries)
- No version history / rollback — re-installing replaces in place
- Global manifest (not per-tenant)
## Updating Installed Packages
Update flow is **Phase 1 GitHub-only** (pip/npm/apk deferred to Phase 2).
### UI
The Runtime & Packages page renders a summary bar above the GitHub Binaries
section when updates are available:
```
┌─────────────────────────────────────────────────────────┐
│ 🟡 3 updates available │
│ Last checked 5m ago [Refresh] [Update All] │
└─────────────────────────────────────────────────────────┘
```
Per-row `[Update]` buttons appear next to each package with a newer release.
Clicking applies the update via atomic `.bak` swap with automatic rollback on
failure.
### API
All write endpoints require **master-scope admin** (tenant admins are denied):
| Endpoint | Purpose |
|---|---|
| `GET /v1/packages/updates` | Cache snapshot + `{stale, ageSeconds, ttlSeconds}` (operator+) |
| `POST /v1/packages/updates/refresh` | Force sync CheckAll — fetch from GitHub |
| `POST /v1/packages/update` | Apply one: body `{"package":"github:lazygit","toVersion":"v0.44.5"}` |
| `POST /v1/packages/updates/apply-all` | Sequential apply; body `{"packages":[...]}` (empty = all). Always returns 200 — inspect `failed[]` |
### Behaviour
- **Stale-while-revalidate**: `GET /updates` returns the cached snapshot
immediately and triggers a background refresh if the cache is older than
`packages.updates_check_ttl` (default `1h`).
- **ETag**: responses use `If-None-Match`, so repeated checks cost zero
rate-limit budget (304 responses don't count against 60/hr).
- **Pre-releases**: if your current tag matches `(-alpha|-beta|-rc|-pre|-preview|-dev|-nightly)`,
the checker polls both `/releases/latest` and `/releases?per_page=5` and
picks the newest via `golang.org/x/mod/semver.Compare`. This correctly
handles the `v1.0.0-rc.1 → v1.0.0` stable transition.
- **Non-semver tags** (e.g. `2024-01-15`): string-compare fallback. Never
downgrades — if the candidate string is lexically less than current, the
update is suppressed.
- **Atomic swap**: two-phase rename. Phase A renames ALL current binaries to
`{name}.bak.{unixNano}`; Phase B renames the new binaries in place. On any
failure during Phase B, Phase A's renames are rolled back. Manifest is
persisted AFTER all swaps succeed, with retries (100ms/500ms/1s).
### WebSocket events
Owner clients receive (non-owner master admins use the HTTP API directly):
```
package.update.checked {count, checked_at}
package.update.started {source, name, from_version, to_version}
package.update.succeeded {source, name, from_version, to_version, duration_ms}
package.update.failed {source, name, reason}
```
### Troubleshooting Updates
#### "Binary updated but manifest save failed" (manifestDesynced=true)
The `.bak` files are deleted but the manifest didn't record the new version.
Next update attempt will re-apply the same version. Manual recovery is not
required — just run the update again OR restart the gateway (which re-reads
the manifest). No data loss.
#### Corrupt updates cache
Symptom: UI shows no updates available despite newer releases.
Recovery: delete `/app/data/.runtime/updates-cache.json`, click `[Refresh]`.
#### Rate-limit exhaustion
Symptom: `Refresh` returns 429 or check returns partial results.
Check response header `X-RateLimit-Reset` (Unix epoch). Wait or set
`packages.github_token` in config (Phase 2 auth — unwired in Phase 1).
#### Scratch dir leftover after crash
Path: `{BinDir}/../tmp/{name}-{tag}-{nanos}/`
Safe to remove any `{name}-*-*` directory under tmp after ensuring no active
update is in flight. Phase 2 will add startup GC.
#### Mid-swap process crash
Phase 1 leaves `.bak.{nanos}` files on disk. Manual recovery:
1. Check `{BinDir}` for `*.bak.*` files.
2. If the main binary is MISSING, rename the `.bak.{nanos}` back to the
original name.
3. If the main binary EXISTS but is the new version you wanted, delete the
`.bak.{nanos}`.
4. Re-run the update via UI — idempotent.
## See Also
- [`docs/packages-pip-npm.md`](./packages-pip-npm.md) — pip + npm package updates (Phase 2a)
- [`docs/14-skills-runtime.md`](./14-skills-runtime.md) — Overview of the runtime packages system
- Issue [#741](https://github.com/nextlevelbuilder/goclaw/issues/741) — Original feature request
docs/packages-pip-npm.md
+196
View File
@@ -0,0 +1,196 @@
# pip + npm Package Updates (Phase 2a)
Extends the Phase 1 GitHub binary update flow to system-wide pip and npm packages.
Closes #900 (Phase 2a).
See also: [GitHub binary updates](./packages-github.md) · [apk system package updates](./packages-apk.md)
---
## Overview
When the gateway is running in Standard edition with `pip3` and/or `npm` on PATH,
`GET /v1/packages/updates` includes pip and npm update results alongside GitHub
binaries. The UI shows a per-source pill filter; sources without a binary on PATH
are hidden automatically.
pip scope: **system-wide** (`--break-system-packages`). pip venv / user-site is not
supported in Phase 2a.
npm scope: **global** (`--global`). Per-project `node_modules` are not touched.
---
## Command Matrix
| Source | Check command | Update command | Check timeout | Update timeout |
|--------|---------------|----------------|---------------|----------------|
| pip | `pip3 list --outdated --format json --break-system-packages` | `pip3 install --upgrade --no-cache-dir --break-system-packages --upgrade-strategy only-if-needed <name>` | 30 s | 5 min |
| npm | `npm outdated --global --json` | `npm install --global <name>@<version>` | 30 s | 5 min |
Pre-release pip check appends `--pre` in a secondary call (see Pre-Release Handling).
---
## Behavior
### pip
- `pip3 list --outdated --format json` emits a JSON array; each element has
`name`, `version`, `latest_version`, `latest_filetype`.
- Exit code is always 0 whether or not updates exist.
- stderr is classified via `ClassifyPipStderr` into sentinel errors (see Error Classes).
### npm
npm's exit-code semantics are non-standard:
| Condition | Exit code | Interpretation |
|-----------|-----------|----------------|
| No outdated packages | 0 | No updates |
| Outdated packages found | 1 | Updates — parse JSON stdout |
| Real npm error (ERESOLVE, network, etc.) | 1 | stderr contains `npm ERR!` |
| Ambiguous (exit 1, no stdout, no stderr) | 1 | Treated as no-updates |
The checker inspects exit code **and** stderr for `npm ERR!` before deciding
whether exit 1 means "updates available" or "real error".
---
## Pre-Release Handling
### pip
Two-call merge strategy:
1. **Primary call** (stable only, no `--pre`): baseline list of outdated packages.
2. If any currently-installed package has a pre-release version (`IsPipPreRelease()`):
**Secondary call** with `--pre` to surface the best available upgrade target.
3. Results are merged by package name; when a name appears in both, the entry
with the lexicographically higher `latest_version` wins.
Gate: `IsPipPreRelease` matches PEP 440 patterns — `a`, `b`, `rc`, `dev`, `.pre`,
`.preview` (case-insensitive, digits optional).
### npm
Single-call strategy with a skip gate:
- If `latest` contains an npm pre-release label
(`-alpha`, `-beta`, `-rc`, `-pre`, `-preview`, `-dev`, `-nightly`, `-snapshot`)
**and** `current` does not → entry is skipped.
- If `current` is already a pre-release and `latest` is too → entry kept (user on
pre-release channel receives the newest pre-release update).
This prevents unexpected upgrades from stable channels to unstable channels.
---
## Availability Detection
A source is considered **available** when two gates both pass:
1. **Binary present**: `exec.LookPath("pip3")` / `exec.LookPath("npm")` succeeds.
2. **Edition allows it**: `edition.Current().SupportsPipNpm == true` (always true
for Standard; always false for Lite desktop).
When a source is unavailable:
- Its checker returns `UpdateCheckResult{Available: false}` (no error, no updates).
- `UpdateRegistry.Availability()` maps that source to `false`.
- `GET /v1/packages/updates` response includes `"availability": {"pip": false, "npm": false}`.
- The frontend hides that source from the filter bar.
Lite edition: `gateway_packages_wiring.go` checks `edition.Current().SupportsPipNpm`
before calling `RegisterChecker` / `RegisterExecutor`. Pip and npm checkers are
never instantiated — `registry.Sources()` returns `["github"]` only.
---
## Error Classes
Sentinel errors are defined in `internal/skills/pkg_update_helpers.go`.
### pip sentinels
| Sentinel | Trigger pattern in stderr | i18n key |
|----------|--------------------------|----------|
| `ErrUpdatePipExternallyManaged` | `externally-managed-environment` / `EXTERNALLY-MANAGED` | `packages.update.pip.externally_managed` |
| `ErrUpdatePipPermission` | `Permission denied` / `EACCES` | `packages.update.pip.permission` |
| `ErrUpdatePipNotFound` | `No matching distribution` / `Could not find a version` | `packages.update.pip.not_found` |
| `ErrUpdatePipNetwork` | `Read timed out` / `ConnectionError` / `network` | `packages.update.pip.network` |
| `ErrUpdatePipConflict` | `incompatible` / `dependency resolver` / `Shallow backtracking` | `packages.update.pip.conflict` |
### npm sentinels
| Sentinel | Trigger pattern in stderr | i18n key |
|----------|--------------------------|----------|
| `ErrUpdateNpmPermission` | `EACCES` | `packages.update.npm.permission` |
| `ErrUpdateNpmConflict` | `ERESOLVE` | `packages.update.npm.conflict` |
| `ErrUpdateNpmNetwork` | `ETIMEDOUT` / `ENOTFOUND` / `getaddrinfo` | `packages.update.npm.network` |
| `ErrUpdateNpmTargetMissing` | `ETARGET` | `packages.update.npm.target_missing` |
| `ErrUpdateNpmNotFound` | `E404` / `404` / `not in this registry` | `packages.update.npm.not_found` |
Unclassified stderr returns a generic wrapped error with a truncated reason
(≤ 500 chars, ANSI-stripped).
---
## Runbook
| Symptom | Fix |
|---------|-----|
| **pip EACCES** — gateway lacks write to site-packages | Run gateway as an owner of `/usr/lib/python3/dist-packages`, or set `PIP_TARGET=/app/data/.pip` + add it to `PYTHONPATH` |
| **npm EACCES** — global prefix owned by root | `npm config set prefix ~/.npm-global`; add `~/.npm-global/bin` to `PATH` in entrypoint |
| **npm ERESOLVE** — peer conflict blocks install | SSH into container: `npm install -g <name>@<version> --legacy-peer-deps`; re-check will clear the entry |
| **pip externally-managed (PEP 668)** | Set env var `PIP_BREAK_SYSTEM_PACKAGES=1`, or upgrade pip to ≥ 23.3 (respects the CLI flag without the env var) |
---
## Minimum Versions
| Runtime | Minimum | Recommended | Notes |
|---------|---------|-------------|-------|
| pip | 20.0 | ≥ 23.3 | `--format json` requires 20+; `--break-system-packages` without env var requires 23.3+ |
| npm | 6.0 | ≥ 10 | Older versions may not emit JSON exit 1 correctly |
| Node.js | 12 | ≥ 18 LTS | npm 10 requires Node 18+ |
---
## Shared Locker
`InstallSingleDep` (skill dep install) and `PipUpdateExecutor.Update` / `NpmUpdateExecutor.Update`
(update apply) share a single `PackageLocker` instance injected via `SetSharedPackageLocker`.
This means concurrent `pip install requests` (from a skill) and `pip upgrade requests`
(from the update flow) are serialized by the same per-key mutex. The lock key is
the bare package name (e.g. `"requests"`) scoped to the source (`"pip"` or `"npm"`).
Operators must not bypass the gateway and call `pip install` directly in parallel
with gateway operations — doing so defeats the shared lock and risks a partial-install
race.
---
## Fixture Regeneration
Test fixtures capture `pip3 list --outdated --format json` and `npm outdated -g --json`
output. When the environment's package versions change, regenerate them:
```bash
# pip fixture — include pip version in filename for drift tracking
pip3 --version # e.g., pip 24.0
pip3 list --outdated --format json --break-system-packages \
> internal/skills/testdata/pip_outdated_pip24.json
# npm fixture — include npm version in filename
npm --version # e.g., 10.5.0
npm outdated --global --json \
> internal/skills/testdata/npm_outdated_npm10.json
# Note: npm exits 1 when packages are outdated — that's expected.
# Update test cases to reference the new filename and expected values.
```
Fixture files are version-stamped in their names so drift between CI environments
is detectable by `git diff`.
docs/project-changelog.md
+231
View File
@@ -4,6 +4,202 @@ Significant changes, features, and fixes in reverse chronological order.
---
## 2026-05-18
### Packages: GitHub installer runtime path
**Fixes**
- Fixed GitHub Releases package installs on bare-metal gateways by defaulting the GitHub binary directory to `{runtimeDir}/bin` instead of Docker-only `/app/data/.runtime/bin`.
- The fix covers installs such as `github:nextlevelbuilder/goclaw-cli@v0.4.1` on the VPS, where `/app` is not writable or present.
**Tests**
- Added default-path regression coverage and made Unix-socket apk helper tests skip cleanly on Windows environments that cannot bind Unix sockets.
---
### Tools: built-in wait delay
**Features**
- Added a built-in `wait` tool with bounded millisecond delays, cancellation support, per-agent min/max settings, and runtime policy visibility.
- Preserved same-response ordering by making `wait` a sequential tool-call barrier.
- Added Web agent settings controls so per-agent wait limits are not dropped on save.
**Tests**
- Added focused wait validation, cancellation, policy, builtin seed, config parsing, and tool-stage ordering coverage.
---
### Providers: ChatGPT OAuth GPT-5.5 default
**Changed**
- Updated ChatGPT Subscription (OAuth) default model, provider model catalog, reasoning metadata, and docs examples to prefer `gpt-5.5`.
**Tests**
- Updated focused provider/model catalog, reasoning, registry, and token-window coverage for `gpt-5.5`.
---
### Deployment: Codex CLI service-user auth
**Fixes**
- Fixed agent-controlled Codex CLI auth on the VPS by ensuring the `goclaw` systemd service user has the ChatGPT login auth file under `/var/lib/goclaw/.codex/auth.json`.
- Documented the required service-user check: `sudo -u goclaw -H codex login status`.
---
### Packages: npm workspace protocol fallback
**Fixes**
- Fixed Node package installs for registry packages published with `workspace:` dependency ranges, such as `@agenttasks/cli`.
- GoClaw now retries npm `EUNSUPPORTEDPROTOCOL workspace:` failures by packing the registry tarball, rewriting workspace dependency ranges to published package versions, and installing the sanitized package folder.
**Tests**
- Added focused coverage for workspace protocol detection and package.json dependency rewrite behavior.
---
## 2026-05-17
### Skills: agent manage grants
**Fixes**
- Added explicit per-agent skill manage grants so agents can edit/delete skills they were authorized to maintain even when `owner_id` no longer matches their current actor identity.
- Auto-granted manage permission to the creating/publishing agent for new managed skills.
**UI**
- Show custom skill owner IDs in the Skills table.
- Added Skills page controls to grant agent skill access and edit/delete permission.
**Tests**
- Added PG/SQLite grant coverage and verified Go builds plus Web UI build.
---
### Agents: provider switch save fix
**Fixes**
- Fixed agent detail save after switching provider/model when the UI clears stale ChatGPT OAuth routing; typed JSON nulls now coerce to `{}` for NOT NULL agent JSON config columns in PostgreSQL and SQLite.
**Tests**
- Added regression coverage for typed `json.RawMessage(nil)` / JSON `null` agent config updates.
---
### Deployment: VPS hybrid GoClaw setup
**Operations**
- Deployed GoClaw to a VPS using bare-metal `systemd` gateway plus Dockerized PostgreSQL 18 pgvector.
- Restored the latest private PostgreSQL backup, then upgraded schema from `57` to `65`.
- Installed Node.js 22 and Codex CLI on the host; interactive `codex --login` remains manual.
- Configured Cloudflare-proxied deployment domain and issued SSL through Certbot/Nginx.
- Added `goclaw-backup-r2.timer` to dump PostgreSQL every 6 hours, upload to private Cloudflare R2 storage, and retain the latest 20 backups.
- Added deployment runbook in `docs/deployment-guide.md`.
**Features**
- Added a protected gateway upgrade HTTP API that triggers the fixed host-local upgrade script asynchronously.
- Added `scripts/goclaw-upgrade-release.sh` and installed the VPS copy at `/usr/local/bin/goclaw-upgrade-release`; dry-run verifies the latest stable server release asset and checksum before deploy.
---
### CI/CD: dev branch beta automation
**Features**
- Added a `Dev CI and Beta Release` GitHub Actions workflow for `dev` pushes that runs Go and Web UI checks before publishing a beta prerelease.
- Added semantic-release-style beta version calculation from Conventional Commits, creating `vX.Y.Z-beta.N` tags and prereleases automatically after tests pass.
- Beta automation uploads Linux binaries and publishes `beta` Docker image tags for the same release version, with Docker Hub publishing enabled when credentials are configured.
**Fixes**
- Made beta prerelease publishing independent of a local checkout by passing the repository explicitly to GitHub CLI release commands.
---
### Agent Permissions: channel and workspace matrix
**Features**
- Added `config.permissions.check` so the UI can preview the effective allow/deny decision for an agent, scope, config type, and user.
- Added Permissions UI support for `userId="*"` to grant all members in a selected group scope.
- Documented the cross-channel agent permission matrix, including Zalo group context writes and workspace/context file boundaries.
**Security**
- Protected group context file writes now require a real sender with `context_files` or legacy `file_writer` permission.
- Group file/context/cron permission-store errors now fail closed instead of silently allowing mutation.
- Backend config permission RPCs validate config types and permission values before storing rules.
**Tests**
- Added focused store and context interceptor coverage for permission preview and protected group context writes.
---
### CLI Credentials: per-agent env vars under Packages
**Features**
- Kept `CLI Credentials` as the Packages tab at `/packages?tab=cli-credentials` and preserved the legacy `/cli-credentials` redirect.
- Removed the duplicate standalone `CLI Credentials` item from the left sidebar.
- Added focused coverage for grant env payload semantics and routing contracts.
**Fixes**
- Made agent-grant environment variable controls easier to find by labeling the row action and adding a visible Environment Variables header inside the grant form.
**Security**
- Nested agent-grant get/update/delete/reveal routes now verify the grant belongs to the binary ID in the URL.
- Grant creation now validates both the CLI binary and target agent exist in the authenticated tenant before inserting.
- Grant updates now validate env payloads before scalar writes, preventing partial state changes on 400 responses.
- Runtime env precedence is covered: per-user env overrides per-agent grant env for duplicate keys.
- Credentialed exec now fails closed if per-user env JSON is invalid.
- SQLite add-column migrations for replayed schema snapshots now skip already-present columns.
**Tests**
- Focused backend, store compile, UI unit, and web build validation pass.
- Live PostgreSQL validation skipped because `TEST_DATABASE_URL` is not set.
---
## 2026-05-16
### Agents: per-agent model fallback
**Features**
- Added per-agent `model_fallback` config with ordered provider/model candidates.
- Agent advanced config UI now supports enabling fallback, adding backup provider/model pairs, and drag-and-drop ordering.
- Runtime wraps the resolved agent provider with fallback only for normal agent execution. Explicit provider/model overrides bypass the fallback chain.
**Migrations**
- **PG:** `000065_agent_model_fallback` adds `agents.model_fallback JSONB NOT NULL DEFAULT '{}'`.
- **SQLite:** schema v33 to v34 adds `agents.model_fallback TEXT NOT NULL DEFAULT '{}'`.
**Tests**
- Focused provider, provider resolver, store tests pass. Main app builds in default and `sqliteonly` modes. Web production build passes.
---
## v3.11.3 — 2026-04-26
### Fixes
@@ -154,6 +350,41 @@ Implementation is evidence-backed against the native ChatGPT Responses API event
- Updated `docs/02-providers.md` and `docs/18-http-api.md` to describe the two-strategy model and the compatibility migration.
---
## 2026-04-21
### Webhook fixes (post-review security & idempotency hardening)
**Fixes**
- **K1: Auth context isolation** — Webhook auth middleware now resolves secret/HMAC signature before tenant injection (eliminating 401 due to tenant scope applied too early). Unscoped store methods `GetByHashUnscoped` + `GetByIDUnscoped` added to WebhookStore interface.
- **K7: IP allowlist enforcement** — Inbound webhook calls now check `ip_allowlist` field (CIDR + exact IP) after bearer/HMAC auth. Empty list = allow all (back-compat). Rejected requests return HTTP 403 with log `security.webhook.ip_denied`.
- **K8: HMAC replay protection** — Per-process nonce cache (key = `sha256(tenant_id + "|" + signature_hex)`) with 320s TTL rejects duplicate signatures within the skew window. Single-node caveat documented. Log: `security.webhook.hmac_replay`.
- **K2: `request_payload` canonical shape** — All webhook audit rows now store `{"body_hash":"<hex64>","meta":{...}}` JSON instead of raw bytes. Idempotency checker compares body hashes to detect replays with different payloads (409 Conflict).
- **K3: Body hash extraction** — `extractBodyHash()` now parses canonical audit payload structure (previously had parsing bugs leading to missed hash validation).
- **K9: Invariant test column fix** — Webhook tenant isolation test now references correct schema columns (`encrypted_secret`, `lease_token`).
- **K4: Worker slot drain** — Fixed channel leak in webhook worker that prevented slot release on successful claims. Concurrency now scales properly under load.
- **K5: Lease-token CAS on UpdateStatus** — Stale webhook receivers can no longer overwrite delivery status. Status updates use optimistic concurrency on `lease_token` (UUID), ensuring only the owning worker can mark the call done. Prevents duplicate delivery from slow receivers.
- **K6: HMAC signing key encryption** — Raw secret (from which `hmac_signing_key = hex(SHA-256(secret))` is derived) is now encrypted at rest via AES-256-GCM using `GOCLAW_ENCRYPTION_KEY`. Database compromise no longer = HMAC key compromise. Clients receive plaintext secret once (create/rotate response) and must store securely.
- **K10: Shared rate limiter instance** — Fixed duplicate `webhookLimiter` instantiation causing doubled RPM enforcement. Single limiter now shared across all webhook endpoints.
**Migrations**
- PostgreSQL: Migration `000057` adds `lease_token` column to `webhook_calls`. Migration `000058` adds `encrypted_secret` column to `webhooks`.
- SQLite: Schema v28 includes both new columns.
**Docs**
- `docs/webhooks.md`: Section 3 clarified bearer/HMAC auth contract + IP allowlist behavior. New Section 14 explains encryption at rest, key contract, DB compromise boundary.
- `docs/00-architecture-overview.md`: Section 12 (Webhook Subsystem) updated to mention lease-token CAS semantics and secret encryption.
**Environment**
- `GOCLAW_ENCRYPTION_KEY` is now **required** for webhook HMAC auth. Same key also encrypts LLM provider credentials.
---
## 2026-04-19
### TTS: Gemini provider + ProviderCapabilities schema engine
docs/runbooks/packages-migration-rollback.md
+88
View File
@@ -0,0 +1,88 @@
# Rollback Runbook: packages-cli-credentials-unified-ui (migration 000058)
## Scope
Migration `000058_agent_grants_env_override` adds `encrypted_env BYTEA` to `secure_cli_agent_grants`.
Phase 2 store code (`Get`, `ListByBinary`) SELECTs this column. If the schema is rolled
back while Phase 2 code is still running, every query against that table will 500.
> **WARNING — DESTRUCTIVE ROLLBACK**
> Running `000058` down **permanently discards** all per-grant env override data.
> Every row in `secure_cli_agent_grants` where `encrypted_env IS NOT NULL` will lose
> its encrypted values. **There is no undo after the column is dropped.**
>
> **Mandatory before running down:**
> ```bash
> pg_dump --table=secure_cli_agent_grants "$DATABASE_URL" > grants_env_backup_$(date +%Y%m%d_%H%M%S).sql
> ```
> The down migration emits a RAISE NOTICE with the count of affected rows before dropping.
> Review the count and abort if non-zero unless you have confirmed data loss is acceptable.
**Critical rule: revert app code FIRST, then migrate the schema down.**
---
## PostgreSQL Rollback
### Step 1 — Revert app binary (FIRST)
Deploy previous binary (the one without Phase 2 store changes) to all pods/instances.
Wait for health checks to pass before proceeding.
```bash
# Verify old binary is live and no Phase-2 store queries are executing
kubectl rollout status deployment/goclaw
```
### Step 2 — Migrate schema down
```bash
# Against production database (use your DSN)
./goclaw migrate down 1
# or with explicit DSN:
migrate -database "$DATABASE_URL" -path migrations down 1
```
### Step 3 — Verify
```bash
psql "$DATABASE_URL" -c "\d secure_cli_agent_grants"
# encrypted_env column should be absent
```
---
## SQLite / Desktop (Lite edition) Rollback
SQLite 3.35+ (bundled via modernc.org/sqlite ≥ v1.18) supports `ALTER TABLE … DROP COLUMN`.
The v27 → v26 downgrade path is **not implemented** in `schema.go` migrations map because
golang-migrate is PostgreSQL-only; SQLite versioning is upgrade-only.
### Option A — Clean reinstall (recommended for desktop users)
1. Back up `~/.goclaw/data/goclaw.db`.
2. Install older version of goclaw-lite.
3. Delete `~/.goclaw/data/goclaw.db`.
4. Restart — fresh DB at v24 schema.
### Option B — Manual column drop (advanced)
```bash
sqlite3 ~/.goclaw/data/goclaw.db \
"ALTER TABLE secure_cli_agent_grants DROP COLUMN encrypted_env;"
# Then manually update schema_version row:
sqlite3 ~/.goclaw/data/goclaw.db \
"UPDATE schema_version SET version = 26;"
```
Requires SQLite ≥ 3.35 (check with `sqlite3 --version`).
---
## Phase 2 Guard
Do NOT roll back the schema while Phase 2 or later code is deployed.
The store method `ListByBinary` hardcodes `encrypted_env` in its SELECT.
Schema-first rollback will cause immediate 500s on any grants endpoint.
docs/webhooks.md
+735
View File
@@ -0,0 +1,735 @@
# Webhook API Reference
> **Authoritative integration guide.** Describes inbound auth, endpoint contracts, outbound callback semantics, retry schedule, and security constraints.
## Table of Contents
1. [Overview](#1-overview)
2. [Admin CRUD](#2-admin-crud)
3. [Authentication](#3-authentication)
4. [Endpoint: POST /v1/webhooks/llm](#4-post-v1webhooksllm)
5. [Endpoint: POST /v1/webhooks/message](#5-post-v1webhooksmessage)
6. [Idempotency](#6-idempotency)
7. [Outbound Callbacks](#7-outbound-callbacks)
8. [Channel Capability Matrix](#8-channel-capability-matrix)
9. [Rate Limits](#9-rate-limits)
10. [Edition Differences](#10-edition-differences)
11. [Security](#11-security)
12. [HMAC Receiver Examples](#12-hmac-receiver-examples)
13. [Audit Payload Shape](#13-audit-payload-shape-webhook_callsrequest_payload)
14. [Encryption at Rest](#14-encryption-at-rest)
---
## 1. Overview
GoClaw webhooks let external systems trigger agents or deliver messages through connected channels. Two webhook kinds exist:
| Kind | Endpoint | Purpose | Editions |
|------|----------|---------|----------|
| `llm` | `POST /v1/webhooks/llm` | Invoke an agent with a user prompt (sync or async) | Standard + Lite |
| `message` | `POST /v1/webhooks/message` | Send a message to a user on a channel | Standard only |
Webhooks are tenant-scoped registry entries. Admins create them via the CRUD API; callers use the returned bearer token or HMAC signing key to authenticate inbound requests.
---
## 2. Admin CRUD
All admin endpoints require tenant-admin role. Bearer token authentication via `Authorization: Bearer <admin-token>`.
### Create — `POST /v1/webhooks`
```json
{
"name": "my-integration",
"kind": "llm",
"agent_id": "<uuid>",
"require_hmac": false,
"localhost_only": false,
"rate_limit_per_min": 60,
"scopes": [],
"ip_allowlist": []
}
```
Fields:
| Field | Type | Required | Notes |
|-------|------|----------|-------|
| `name` | string | yes | Max 100 chars |
| `kind` | string | yes | `"llm"` or `"message"` |
| `agent_id` | UUID | for `llm` kind | Agent to invoke |
| `channel_id` | UUID | optional | Pin webhook to a specific channel instance (message kind) |
| `require_hmac` | bool | no | Force HMAC-only auth (disable bearer) |
| `localhost_only` | bool | no | Restrict callers to 127.0.0.1/::1. Auto-set on Lite edition |
| `rate_limit_per_min` | int | no | Per-webhook cap; 0 = use tenant default |
| `scopes` | []string | no | Reserved for future scope enforcement |
| `ip_allowlist` | []string | no | Allowlist of IPs or CIDR ranges. Empty = allow all. See [IP Allowlist](#ip-allowlist) |
**Response — 201 Created**
```json
{
"id": "<uuid>",
"tenant_id": "<uuid>",
"agent_id": "<uuid>",
"name": "my-integration",
"kind": "llm",
"secret_prefix": "wh_ABCD",
"secret": "wh_ABCDEFGHIJKLMNOPQRSTUVWXYZ234567ABCDEFGH",
"hmac_signing_key": "a3f4...hex64chars",
"scopes": [],
"rate_limit_per_min": 60,
"ip_allowlist": [],
"require_hmac": false,
"localhost_only": false,
"created_at": "2026-04-21T12:00:00Z"
}
```
**`secret` and `hmac_signing_key` are returned exactly once — on create and rotate. Store them securely; they cannot be retrieved again.**
- `secret` — raw bearer token. Send as `Authorization: Bearer wh_...`
- `hmac_signing_key` — `hex(SHA-256(secret))`. Used as the HMAC signing key for `X-GoClaw-Signature`. To sign: `HMAC_SHA256(key=hex.Decode(hmac_signing_key), payload="{ts}.{body}")`
### List — `GET /v1/webhooks`
Query params: `agent_id=<uuid>` (optional filter).
Returns array of webhook objects. `secret` and `hmac_signing_key` are **not** included.
### Get — `GET /v1/webhooks/{id}`
Returns full webhook object (no secret).
### Update — `PATCH /v1/webhooks/{id}`
Partial update. All fields optional. Cannot change `kind`.
```json
{
"name": "new-name",
"require_hmac": true,
"localhost_only": false
}
```
### Rotate Secret — `POST /v1/webhooks/{id}/rotate`
Generates a new secret immediately. **No grace window** — the old secret is invalidated the moment rotate completes. Coordinate with callers before rotating.
**Response — 200 OK**
```json
{
"id": "<uuid>",
"secret": "wh_NEW...",
"hmac_signing_key": "newhex...",
"secret_prefix": "wh_NEWX"
}
```
### Revoke — `DELETE /v1/webhooks/{id}`
Marks the webhook as revoked. All subsequent inbound requests with its secret return `401`. Action is irreversible.
---
## 3. Authentication
Two authentication modes. The webhook row's `require_hmac` field determines which are accepted.
### 3.1 Bearer Auth
```
Authorization: Bearer wh_ABCDEFGHIJKLMNOPQRSTUVWXYZ234567ABCDEFGH
```
The gateway SHA-256 hashes the token and looks up `secret_hash` in the database. Constant-time comparison prevents timing oracle attacks.
Bearer auth is **disabled** when `require_hmac=true` on the webhook row.
### 3.2 HMAC Auth
Recommended for Standard edition integrations. Provides both authentication and payload integrity.
**Required headers:**
```
X-Webhook-Id: <webhook-uuid>
X-GoClaw-Signature: t=<unix_seconds>,v1=<hmac_hex>
Content-Type: application/json
```
**Signing algorithm:**
```
signing_key = hex.Decode(hmac_signing_key) // decode the hex field to raw bytes
payload = "{unix_ts}.{request_body_bytes}"
signature = HMAC_SHA256(key=signing_key, data=payload)
header = "t={unix_ts},v1={hex(signature)}"
```
**Timestamp skew:** The gateway rejects requests where `|now - t| > 300 seconds`. Ensure your clock is synchronized (NTP).
**Key contract:** `hmac_signing_key` = `hex(SHA-256(raw_secret))`. The signing key is the **decoded bytes** of this hex string. The raw secret is never stored — only its hash.
### HMAC Replay Protection
After a valid HMAC signature is accepted, the gateway records `sha256(tenant_id + "|" + signature_hex)` in an in-memory nonce cache with a 320-second TTL (> 2× skew window). Any request replaying the same signature within the window is rejected with HTTP 401 and logged as `security.webhook.hmac_replay`.
**Single-node caveat:** The nonce cache is per-process and not distributed. In a multi-node deployment a replay could succeed on a different node. This is an accepted trade-off for the current single-process gateway architecture.
### IP Allowlist
When `ip_allowlist` is non-empty, the gateway checks the request's source IP (from `RemoteAddr`) against every entry after successful auth. Each entry can be:
- A single IP address: `"1.2.3.4"`, `"::1"`
- A CIDR range: `"10.0.0.0/8"`, `"2001:db8::/32"`
An empty `ip_allowlist` (the default) allows requests from any source — back-compat with existing webhooks.
Rejected requests return HTTP 403 and are logged as `security.webhook.ip_denied`.
**Proxy note:** `X-Forwarded-For` is **not** trusted — only `RemoteAddr` is used. If your gateway sits behind a reverse proxy, ensure the proxy is configured to terminate TLS and handle allowlist enforcement itself, or accept that `RemoteAddr` will be the proxy IP.
---
## 4. POST /v1/webhooks/llm
Triggers an agent with an input prompt. Available in all editions.
**Auth:** Bearer or HMAC (per webhook `require_hmac` setting). Webhook must have `kind="llm"`.
### Request
```json
{
"input": "Summarize the latest metrics",
"session_key": "user-123-session",
"user_id": "ext-user-456",
"model": "claude-opus-4-5",
"mode": "sync",
"callback_url": "",
"metadata": {}
}
```
| Field | Type | Required | Notes |
|-------|------|----------|-------|
| `input` | string or array | yes | Plain string, or `[{role, content}]` array |
| `session_key` | string | no | Stable key for multi-turn conversation continuity |
| `user_id` | string | no | External user identifier for scoping |
| `model` | string | no | Per-request model override |
| `mode` | string | no | `"sync"` (default) or `"async"` |
| `callback_url` | string | required if async | HTTPS URL for delivery. Validated against SSRF policy |
| `metadata` | object | no | Echoed to callback payload (max 8 KB) |
**Input formats:**
```json
// Plain string
"input": "Hello agent"
// Message array
"input": [
{"role": "system", "content": "You are a concise assistant"},
{"role": "user", "content": "List 3 key metrics"}
]
```
### Sync Response — 200 OK
```json
{
"call_id": "<uuid>",
"agent_id": "<uuid>",
"output": "Here are the metrics: ...",
"usage": {
"prompt_tokens": 150,
"completion_tokens": 200,
"total_tokens": 350
},
"finish_reason": "stop"
}
```
Sync mode times out at **30 seconds**. On timeout: `504 Gateway Timeout` with `webhook.llm_timeout`.
### Async Response — 202 Accepted
```json
{
"call_id": "<uuid>",
"status": "queued"
}
```
The agent runs asynchronously. Results are delivered via outbound callback (see [Section 7](#7-outbound-callbacks)).
### Error Responses
| Status | Code | When |
|--------|------|------|
| 400 | `invalid_request` | Missing `input`, bad `mode`, missing `callback_url` for async |
| 401 | — | Auth failure (bearer invalid, HMAC mismatch, revoked, HMAC replay) |
| 403 | `unauthorized` | `localhost_only` violation, IP allowlist denial, kind mismatch, tenant mismatch |
| 404 | `not_found` | Agent not found |
| 429 | — | Rate limit exceeded; `Retry-After: 60` header set |
| 503 | — | Webhook processing lane at capacity |
| 504 | — | LLM timeout (sync mode only) |
---
## 5. POST /v1/webhooks/message
Sends a message to a user on a connected channel. **Standard edition only** — not available on Lite.
**Auth:** Bearer or HMAC (per webhook `require_hmac` setting). Webhook must have `kind="message"`.
### Request
```json
{
"channel_name": "telegram-prod",
"chat_id": "123456789",
"content": "Hello from the integration!",
"media_url": "https://example.com/image.jpg",
"media_caption": "Optional caption",
"fallback_to_text": false
}
```
| Field | Type | Required | Notes |
|-------|------|----------|-------|
| `channel_name` | string | yes (unless webhook has bound `channel_id`) | Channel instance name |
| `chat_id` | string | yes | Channel-specific recipient ID |
| `content` | string | yes (unless `media_url`) | Text body; max 16 KB |
| `media_url` | string | no | HTTPS URL to media file. SSRF-guarded + HEAD-probed |
| `media_caption` | string | no | Caption for media |
| `fallback_to_text` | bool | no | If true, send text-only when channel can't handle media |
### Response — 200 OK
```json
{
"call_id": "<uuid>",
"status": "sent",
"channel_name": "telegram-prod",
"chat_id": "123456789",
"warning": ""
}
```
`warning` is set to `"media_not_supported_fallback_text"` when `fallback_to_text=true` and media was dropped.
### Error Responses
| Status | Code | When |
|--------|------|------|
| 400 | `invalid_request` | Missing `chat_id`, `content`, SSRF-blocked `media_url` |
| 403 | `unauthorized` | Channel belongs to different tenant |
| 404 | `not_found` | Channel instance not found |
| 415 | `invalid_request` | MIME type denied for media |
| 429 | — | Rate limit exceeded |
| 501 | `invalid_request` | Channel does not support media and `fallback_to_text=false` |
---
## 6. Idempotency
All webhook endpoints support idempotency via the `Idempotency-Key` header.
```
Idempotency-Key: <opaque-string-max-255-chars>
```
**Semantics:**
- First request with a given key: processed normally.
- Subsequent requests with the **same key and identical body**: return the cached response immediately with `200 OK` (no duplicate processing).
- Subsequent requests with the **same key but different body**: return `409 Conflict` with `webhook.idempotency_conflict`.
- Keys expire after 24 hours (implementation: `webhook_calls` table TTL).
**Recommendation:** Use a UUID or hash of request content as the key. Re-send the exact same request body on retry.
---
## 7. Outbound Callbacks
Async LLM calls (`mode=async`) deliver results to the `callback_url` via HTTP POST.
### Delivery Guarantee
Callbacks are **at-least-once**. Receivers must be idempotent.
### Stable Headers
Every delivery attempt carries:
```
X-Webhook-Delivery-Id: <uuid> -- stable across retries
X-Webhook-Signature: t=<unix>,v1=<hex> -- recomputed per attempt (timestamp differs)
Content-Type: application/json
User-Agent: goclaw-webhook/1
```
`X-Webhook-Delivery-Id` is stable for all retry attempts of the same call. Receivers **SHOULD** deduplicate by this ID within a window of at least 24 hours.
`X-Webhook-Signature` uses the **same HMAC algorithm** as inbound auth. Verify with the `hmac_signing_key` from the create response.
### Payload
```json
{
"call_id": "<uuid>",
"delivery_id": "<uuid>",
"agent_id": "<uuid>",
"status": "done",
"output": "Agent response text...",
"usage": {
"prompt_tokens": 150,
"completion_tokens": 200,
"total_tokens": 350
},
"metadata": {},
"error": ""
}
```
`status` is `"done"` on success, `"failed"` on agent error. `error` is non-empty on failure.
### Retry Schedule
| Attempt | Delay (±10% jitter) |
|---------|---------------------|
| 1 | 30 seconds |
| 2 | 2 minutes |
| 3 | 10 minutes |
| 4 | 1 hour |
| 5 | 6 hours |
After 5 failed attempts the row moves to `status=dead`. No further retries.
**`Retry-After` header:** If the receiver returns `429` with a `Retry-After` header, the worker respects it (capped at 6 hours).
**Permanent failure:** `4xx` responses (except `429`) are treated as permanent — no retry.
**Success:** Any `2xx` response marks the delivery as done.
### Verifying Outbound Signatures
```go
// Go — verify X-Webhook-Signature on your callback endpoint
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"strconv"
"strings"
"time"
)
func verifyWebhookSignature(r *http.Request, body []byte, hmacSigningKey string) error {
sigHeader := r.Header.Get("X-Webhook-Signature")
// Parse "t=<unix>,v1=<hex>"
var ts int64
var sigHex string
for _, part := range strings.Split(sigHeader, ",") {
if strings.HasPrefix(part, "t=") {
ts, _ = strconv.ParseInt(strings.TrimPrefix(part, "t="), 10, 64)
}
if strings.HasPrefix(part, "v1=") {
sigHex = strings.TrimPrefix(part, "v1=")
}
}
if ts == 0 || sigHex == "" {
return fmt.Errorf("missing signature header fields")
}
// Verify timestamp skew
if abs(time.Now().Unix()-ts) > 300 {
return fmt.Errorf("timestamp skew too large")
}
// Decode HMAC key from hex
key, err := hex.DecodeString(hmacSigningKey)
if err != nil {
return err
}
// Recompute HMAC
payload := append([]byte(fmt.Sprintf("%d.", ts)), body...)
mac := hmac.New(sha256.New, key)
mac.Write(payload)
expected := mac.Sum(nil)
// Decode received sig
received, err := hex.DecodeString(sigHex)
if err != nil || !hmac.Equal(expected, received) {
return fmt.Errorf("signature mismatch")
}
return nil
}
```
---
## 8. Channel Capability Matrix
Relevant for `POST /v1/webhooks/message` with `media_url`.
| Channel Type | Text | Media |
|--------------|------|-------|
| `telegram` | yes | yes |
| `discord` | yes | yes |
| `whatsapp` | yes | yes |
| `feishu` | yes | yes |
| `slack` | yes | yes |
| `zalo_personal` | yes | yes |
| `pancake` | yes | yes |
| `facebook` | yes | yes |
| `zalo_oa` | yes | no |
When `media_url` is sent to a non-media-capable channel:
- `fallback_to_text=true` → text content delivered, `warning` field set
- `fallback_to_text=false` (default) → `501 Not Implemented`
---
## 9. Rate Limits
Rate limiting is two-tier:
| Tier | Cap | Notes |
|------|-----|-------|
| Per-webhook | `rate_limit_per_min` field (0 = disabled) | Configured per webhook row |
| Per-tenant | Platform default (configurable) | Applies across all webhooks for a tenant |
Both tiers must pass. If either rejects the request, `429 Too Many Requests` is returned with `Retry-After: 60`.
---
## 10. Edition Differences
| Feature | Standard | Lite |
|---------|----------|------|
| `/v1/webhooks/llm` | Available | Available (localhost_only forced) |
| `/v1/webhooks/message` | Available | Disabled |
| `localhost_only=false` | Configurable | Always true; cannot be unset |
| `kind="message"` webhook creation | Allowed | Rejected (403) |
On Lite, all webhooks are automatically created with `localhost_only=true` regardless of the request field. Attempting to unset `localhost_only` via PATCH returns `403`.
---
## 11. Security
### SSRF Protection
- `media_url` in message webhooks: validated against SSRF policy + HEAD-probed before fetch.
- `callback_url` in async LLM webhooks: validated at enqueue time and re-validated at delivery time (prevents DNS rebinding attacks).
- Log event: `security.webhook.ssrf_blocked` / `security.webhook.callback_ssrf_blocked`.
### Secret Storage
Secrets are never stored in plaintext. Only `SHA-256(secret)` is kept in the database. Secrets are never logged.
### HMAC Timestamp Skew
Requests with `|now - t| > 300 seconds` are rejected immediately (before any DB lookup) to prevent replay attacks.
### Tenant Isolation
- Agent must belong to the webhook's tenant.
- Channel must belong to the webhook's tenant (or be a legacy config-based channel).
- Log events: `security.webhook.tenant_mismatch`, `security.webhook.tenant_leak_attempt`.
### Secret Rotation
**No grace window.** The old secret is invalidated immediately when `POST /v1/webhooks/{id}/rotate` completes. Coordinate with callers before rotating in production.
---
## 12. HMAC Receiver Examples
### curl (signing with openssl)
```bash
WEBHOOK_HMAC_KEY="a3f4...your_hmac_signing_key_hex"
WEBHOOK_ID="your-webhook-uuid"
BODY='{"input":"hello","mode":"sync"}'
TS=$(date +%s)
PAYLOAD="${TS}.${BODY}"
SIG=$(echo -n "$PAYLOAD" | openssl dgst -sha256 -mac HMAC \
-macopt "hexkey:${WEBHOOK_HMAC_KEY}" | awk '{print $2}')
curl -X POST https://example.com/v1/webhooks/llm \
-H "Content-Type: application/json" \
-H "X-Webhook-Id: ${WEBHOOK_ID}" \
-H "X-GoClaw-Signature: t=${TS},v1=${SIG}" \
-d "$BODY"
```
### curl (bearer auth)
```bash
curl -X POST https://example.com/v1/webhooks/llm \
-H "Authorization: Bearer wh_ABCDEFGHIJKLMNOPQRSTUVWXYZ234567ABCDEFGH" \
-H "Content-Type: application/json" \
-d '{"input":"hi","mode":"sync"}'
```
### Node.js (HMAC signing)
```js
const crypto = require('crypto');
function signWebhookRequest(body, hmacSigningKeyHex) {
const ts = Math.floor(Date.now() / 1000);
const keyBytes = Buffer.from(hmacSigningKeyHex, 'hex');
const payload = Buffer.concat([
Buffer.from(`${ts}.`),
Buffer.isBuffer(body) ? body : Buffer.from(body),
]);
const sig = crypto.createHmac('sha256', keyBytes).update(payload).digest('hex');
return { ts, signature: `t=${ts},v1=${sig}` };
}
// Usage
const body = JSON.stringify({ input: 'hello', mode: 'sync' });
const { signature } = signWebhookRequest(body, process.env.WEBHOOK_HMAC_KEY);
await fetch('https://example.com/v1/webhooks/llm', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Webhook-Id': process.env.WEBHOOK_ID,
'X-GoClaw-Signature': signature,
},
body,
});
```
### Python (HMAC signing)
```python
import hashlib
import hmac
import json
import time
import requests
def sign_webhook(body: bytes, hmac_signing_key_hex: str) -> str:
ts = int(time.time())
key = bytes.fromhex(hmac_signing_key_hex)
payload = f"{ts}.".encode() + body
sig = hmac.new(key, payload, hashlib.sha256).hexdigest()
return f"t={ts},v1={sig}"
body = json.dumps({"input": "hello", "mode": "sync"}).encode()
signature = sign_webhook(body, os.environ["WEBHOOK_HMAC_KEY"])
requests.post(
"https://example.com/v1/webhooks/llm",
headers={
"Content-Type": "application/json",
"X-Webhook-Id": os.environ["WEBHOOK_ID"],
"X-GoClaw-Signature": signature,
},
data=body,
)
```
---
## 13. Audit Payload Shape (`webhook_calls.request_payload`)
Every call creates a row in `webhook_calls` with a `request_payload` column (`jsonb` on PostgreSQL, `TEXT` on SQLite). The canonical shape is:
```json
{
"body_hash": "<sha256-hex-64-chars>",
"meta": { ... handler-specific fields ... }
}
```
### `body_hash`
SHA-256 hex digest of the raw request body bytes. Used by the idempotency subsystem to detect body-mismatch replays (same `Idempotency-Key`, different body → 409 Conflict).
### `meta` by handler
**`POST /v1/webhooks/llm`** — meta mirrors the decoded request fields:
```json
{
"input": "<raw JSON — string or message array>",
"session_key": "optional-key",
"user_id": "optional-uid",
"model": "optional-override",
"mode": "sync",
"callback_url": "",
"metadata": null
}
```
**`POST /v1/webhooks/message`** — meta contains delivery context:
```json
{
"channel_name": "telegram-main",
"chat_id": "123456789",
"has_media": false
}
```
### Notes
- `body_hash` is always exactly 64 lowercase hex characters. Any stored value that does not match this format is treated as "no hash" by the idempotency checker (fail-closed).
- External consumers reading `request_payload` via SQL should parse it as JSON, not as raw bytes.
- Shape is stable across LLM and message handler calls — only `meta` contents differ.
---
## 14. Encryption at Rest
### Raw Secret Encryption
The webhook secret is encrypted at rest using AES-256-GCM, keyed by the environment variable `GOCLAW_ENCRYPTION_KEY` (required for webhook HMAC auth to work). Only the database stores encrypted secret material.
**Key contract (POST /v1/webhooks create/rotate response):**
```json
{
"secret": "wh_ABCDEFGHIJKLMNOPQRSTUVWXYZ234567ABCDEFGH",
"hmac_signing_key": "a3f4...hex64chars"
}
```
- `secret` — Raw bearer token in plaintext. Clients **must store securely** on their end; the gateway will not retrieve it again.
- `hmac_signing_key` — Derived as `hex(SHA-256(secret))`. This is also returned once and should be stored securely by clients.
**Database storage:**
- `webhooks.secret_hash` column: `SHA-256(secret)` in hex. Used for bearer auth lookups (constant-time comparison).
- `webhooks.encrypted_secret` column (PG/SQLite): AES-256-GCM encrypted raw secret. Used to support lease-token reclamation and idempotency recovery on stale calls.
- Environment variable `GOCLAW_ENCRYPTION_KEY` — required for webhook processing. Same key also encrypts LLM provider API keys. Format: base64-encoded 32-byte key.
**Migration notes:**
- PostgreSQL: Migration `000058` added `encrypted_secret` column.
- SQLite (Lite edition): Schema v28 includes encrypted secret support.
**DB compromise impact:**
A database-layer attacker with read-only access to `webhooks` table **cannot** derive the raw secret or `hmac_signing_key`:
- `secret_hash` alone does not reverse-engineer the secret (cryptographic hash).
- `encrypted_secret` requires `GOCLAW_ENCRYPTION_KEY` to decrypt (environment-only, not in database).
- Attackers gain no actionable HMAC material.
### Environment Variable Security
`GOCLAW_ENCRYPTION_KEY` must be:
- Stored securely (e.g., sealed in a secret manager, not in `config.json`).
- Same across all gateway instances in a cluster (standard multi-replica key).
- Rotated as part of incident response — rotation requires re-encrypting all webhook secrets (automated migration).
---
go.mod
+4 -2
View File
@@ -42,6 +42,8 @@ require (
go.opentelemetry.io/otel/sdk v1.40.0
go.opentelemetry.io/otel/trace v1.40.0
golang.org/x/image v0.27.0
golang.org/x/mod v0.35.0
golang.org/x/oauth2 v0.34.0
golang.org/x/time v0.14.0
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.47.0
@@ -50,6 +52,7 @@ require (
require (
cel.dev/expr v0.25.1 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/akutz/memconn v0.1.0 // indirect
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect
@@ -154,7 +157,6 @@ require (
go.uber.org/atomic v1.11.0 // indirect
go4.org/mem v0.0.0-20240501181205-ae6ca9944745 // indirect
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
golang.org/x/oauth2 v0.34.0 // indirect
golang.org/x/term v0.40.0 // indirect
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
golang.zx2c4.com/wireguard/windows v0.5.3 // indirect
@@ -202,7 +204,7 @@ require (
go.opentelemetry.io/otel/metric v1.40.0 // indirect
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
golang.org/x/crypto v0.48.0 // indirect
golang.org/x/crypto v0.48.0
golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect
golang.org/x/net v0.50.0
golang.org/x/sync v0.19.0
go.sum
+6 -4
View File
@@ -2,6 +2,8 @@
9fans.net/go v0.0.8-0.20250307142834-96bdba94b63f/go.mod h1:hHyrZRryGqVdqrknjq5OWDLGCTJ2NeEvtrpR96mjraM=
cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
filippo.io/mkcert v1.4.4 h1:8eVbbwfVlaqUM7OwuftKc2nuYOoTDQWqsoXmzoXZdbc=
@@ -585,8 +587,8 @@ golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:AbB0pIl
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.27.0 h1:C8gA4oWU/tKkdCfYT6T2u4faJu3MeNS5O8UPWlPF61w=
golang.org/x/image v0.27.0/go.mod h1:xbdrClrAUway1MUTEZDq9mz/UpRwYAkFFNUslZtcB+g=
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM=
golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
@@ -617,8 +619,8 @@ golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg=
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
internal/agent/loop_context.go
+10
View File
@@ -108,6 +108,11 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
if l.memoryCfg != nil {
ctx = tools.WithMemoryConfig(ctx, l.memoryCfg)
}
var waitToolCfg *config.WaitToolPolicy
if l.agentToolPolicy != nil && l.agentToolPolicy.Wait != nil {
waitToolCfg = l.agentToolPolicy.Wait
ctx = tools.WithWaitToolConfig(ctx, waitToolCfg)
}
if l.sandboxCfg != nil {
ctx = tools.WithSandboxConfig(ctx, l.sandboxCfg)
}
@@ -156,6 +161,9 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
}
// Apply user isolation layer via pipeline.
shared := l.shouldShareWorkspace(req.UserID, req.PeerKind)
if shared {
ctx = store.WithSharedContext(ctx)
}
effectiveWorkspace := tools.ResolveWorkspace(ws,
tools.UserChatLayer(tools.SanitizePathSegment(req.UserID), shared),
)
@@ -359,6 +367,7 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
SharedMemory: store.IsSharedMemory(ctx),
SharedKG: store.IsSharedKG(ctx),
SharedSessions: store.IsSharedSessions(ctx),
SharedContext: store.IsSharedContext(ctx),
RestrictToWorkspace: l.restrictToWs != nil && *l.restrictToWs,
BuiltinToolSettings: l.builtinToolSettings,
ChannelType: req.ChannelType,
@@ -367,6 +376,7 @@ func (l *Loop) injectContext(ctx context.Context, req *RunRequest) (contextSetup
ParentProvider: providerName,
MemoryCfg: l.memoryCfg,
SandboxCfg: l.sandboxCfg,
WaitToolCfg: waitToolCfg,
ShellDenyGroups: l.shellDenyGroups,
Workspace: tools.ToolWorkspaceFromCtx(ctx),
TeamWorkspace: tools.ToolTeamWorkspaceFromCtx(ctx),
internal/agent/loop_pipeline_adapter.go
+8 -1
View File
@@ -27,6 +27,10 @@ func (l *Loop) runViaPipeline(ctx context.Context, req RunRequest) (*RunResult,
provider := l.provider
if req.ProviderOverride != nil {
provider = req.ProviderOverride
} else if req.ModelOverride != "" {
if fallback, ok := provider.(interface{ PrimaryProvider() providers.Provider }); ok {
provider = fallback.PrimaryProvider()
}
}
p := pipeline.NewDefaultPipeline(deps)
@@ -137,7 +141,10 @@ func (l *Loop) buildPipelineDeps(req *RunRequest, bridgeRS *runState) pipeline.P
ExecuteToolCall: cb.executeToolCall,
ExecuteToolRaw: cb.executeToolRaw,
ProcessToolResult: cb.processToolResult,
CheckReadOnly: cb.checkReadOnly,
SequentialToolCall: func(tc providers.ToolCall) bool {
return l.resolveToolCallName(tc.Name) == "wait"
},
CheckReadOnly: cb.checkReadOnly,
// Observe: drain InjectCh
DrainInjectCh: func() []providers.Message {
internal/agent/loop_utils_test.go
+8 -6
View File
@@ -1,6 +1,7 @@
package agent
import (
"path/filepath"
"strings"
"testing"
@@ -134,7 +135,7 @@ func TestShouldShareSessions_DisabledByDefault(t *testing.T) {
func TestShouldShareSessions_IndependentOfMemory(t *testing.T) {
l := &Loop{workspaceSharing: &store.WorkspaceSharingConfig{
ShareMemory: true,
ShareMemory: true,
ShareSessions: false,
}}
if l.shouldShareSessions() {
@@ -181,9 +182,10 @@ func TestProviderName_WithProvider(t *testing.T) {
// ─── expandWorkspace ──────────────────────────────────────────────────────
func TestExpandWorkspace_AbsolutePathUnchanged(t *testing.T) {
got := expandWorkspace("/absolute/path")
if got != "/absolute/path" {
t.Errorf("expandWorkspace = %q, want /absolute/path", got)
abs := filepath.Join(t.TempDir(), "absolute", "path")
got := expandWorkspace(abs)
if got != filepath.Clean(abs) {
t.Errorf("expandWorkspace = %q, want %q", got, filepath.Clean(abs))
}
}
@@ -192,14 +194,14 @@ func TestExpandWorkspace_HomeExpanded(t *testing.T) {
if strings.HasPrefix(got, "~") {
t.Errorf("tilde not expanded: %q", got)
}
if !strings.HasPrefix(got, "/") {
if !filepath.IsAbs(got) {
t.Errorf("expected absolute path after ~ expansion, got %q", got)
}
}
func TestExpandWorkspace_RelativePathBecomesAbsolute(t *testing.T) {
got := expandWorkspace("relative/path")
if !strings.HasPrefix(got, "/") {
if !filepath.IsAbs(got) {
t.Errorf("relative path should become absolute, got %q", got)
}
}
internal/agent/resolver.go
+3 -3
View File
@@ -15,9 +15,9 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/eventbus"
"github.com/nextlevelbuilder/goclaw/internal/hooks"
"github.com/nextlevelbuilder/goclaw/internal/memory"
mcpbridge "github.com/nextlevelbuilder/goclaw/internal/mcp"
"github.com/nextlevelbuilder/goclaw/internal/media"
"github.com/nextlevelbuilder/goclaw/internal/memory"
"github.com/nextlevelbuilder/goclaw/internal/providerresolve"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/sandbox"
@@ -158,7 +158,7 @@ func NewManagedResolver(deps ResolverDeps) ResolverFunc {
}
// Resolve provider (tenant-aware: tries tenant-specific first, falls back to master)
provider, err := providerresolve.ResolveConfiguredProvider(deps.ProviderReg, ag)
provider, err := providerresolve.ResolveAgentProvider(deps.ProviderReg, ag)
if err != nil {
// Fallback to any available provider for this tenant
names := deps.ProviderReg.ListForTenant(ag.TenantID)
@@ -467,7 +467,7 @@ func NewManagedResolver(deps ResolverDeps) ResolverFunc {
AgentOtherConfig: ag.OtherConfig,
AgentType: ag.AgentType,
IsTeamLead: isTeamLead,
AutoInjector: deps.AutoInjector,
AutoInjector: deps.AutoInjector,
Provider: provider,
Model: ag.Model,
ModelRegistry: deps.ModelRegistry,
internal/agent/toolloop.go
+3 -2
View File
@@ -142,7 +142,7 @@ func (s *toolLoopState) detect(toolName string, argsHash string) (level, message
}
// recordMutation updates the read-only streak based on tool type.
// Mutating tools reset the streak; exec/bash/mcp are neutral (ambiguous); all others increment.
// Mutating tools reset the streak; exec/bash/wait/mcp are neutral; all others increment.
// team_tasks is classified by action: read-only (list/get/search), neutral (progress),
// or mutating (create/complete/cancel/comment/etc.).
func (s *toolLoopState) recordMutation(toolName string, args map[string]any) {
@@ -168,9 +168,10 @@ func (s *toolLoopState) recordMutation(toolName string, args map[string]any) {
return
}
// exec/bash: ambiguous (could be ls or rm).
// wait: intentional delay, neither progress nor read-only scanning.
// mcp_*: user-defined external tools — GoClaw cannot determine read vs write.
// Neither reset nor increment the read-only streak.
if toolName == "exec" || toolName == "bash" || strings.HasPrefix(toolName, "mcp_") {
if toolName == "exec" || toolName == "bash" || toolName == "wait" || strings.HasPrefix(toolName, "mcp_") {
return
}
s.incrementReadOnly(toolName, args)
internal/agent/toolloop_test.go
+11
View File
@@ -570,6 +570,17 @@ func TestReadOnlyStreak_ExecNeutral(t *testing.T) {
}
}
func TestReadOnlyStreak_WaitNeutral(t *testing.T) {
var s toolLoopState
for range 5 {
s.recordMutation("read_file", nil)
}
s.recordMutation("wait", map[string]any{"timeMs": 1000})
if s.readOnlyStreak != 5 {
t.Fatalf("expected streak 5 after wait, got %d", s.readOnlyStreak)
}
}
func TestReadOnlyStreak_MCPNeutral(t *testing.T) {
var s toolLoopState
// 5 reads → streak = 5
internal/backup/pgpass_test.go
+4
View File
@@ -3,6 +3,7 @@ package backup
import (
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
@@ -59,6 +60,9 @@ func TestParseDSN_SpecialCharsInPassword(t *testing.T) {
}
func TestWritePgpass_FilePermissions(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Windows does not expose POSIX 0600 permissions reliably")
}
creds := &PGCredentials{
Host: "localhost", Port: "5432",
User: "testuser", Password: "testpass",
internal/bootstrap/backfill_capabilities.go
+1 -2
View File
@@ -4,7 +4,6 @@ import (
"context"
"database/sql"
"log/slog"
"path/filepath"
)
// BackfillCapabilities seeds CAPABILITIES.md template for all agents that don't have it.
@@ -15,7 +14,7 @@ func BackfillCapabilities(ctx context.Context, db *sql.DB) (int64, error) {
return 0, nil
}
tpl, err := templateFS.ReadFile(filepath.Join("templates", CapabilitiesFile))
tpl, err := templateFS.ReadFile(templatePath(CapabilitiesFile))
if err != nil {
return 0, err
}
internal/bootstrap/seed.go
+7 -2
View File
@@ -4,6 +4,7 @@ import (
"embed"
"log/slog"
"os"
"path"
"path/filepath"
)
@@ -25,7 +26,7 @@ var templateFiles = []string{
// ReadTemplate returns the content of an embedded template file.
func ReadTemplate(name string) (string, error) {
content, err := templateFS.ReadFile(filepath.Join("templates", name))
content, err := templateFS.ReadFile(templatePath(name))
if err != nil {
return "", err
}
@@ -88,7 +89,7 @@ func seedTemplate(workspaceDir, name string) (bool, error) {
defer f.Close()
// Read embedded template
content, err := templateFS.ReadFile(filepath.Join("templates", name))
content, err := templateFS.ReadFile(templatePath(name))
if err != nil {
os.Remove(dstPath) // clean up empty file
return false, err
@@ -100,3 +101,7 @@ func seedTemplate(workspaceDir, name string) (bool, error) {
return true, nil
}
func templatePath(name string) string {
return path.Join("templates", name)
}
internal/bootstrap/seed_store.go
+5 -8
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"log/slog"
"path/filepath"
"strings"
"time"
@@ -49,8 +48,6 @@ _(First contact via %s channel. Profile info auto-filled from channel data.)_
`, name, name, tz, meta.ChannelType)
}
// retryOnBusy retries fn up to 3 times on SQLITE_BUSY errors with 500ms delay.
func retryOnBusy(fn func() error) error {
var lastErr error
@@ -110,7 +107,7 @@ func SeedToStore(ctx context.Context, agentStore store.AgentStore, agentID uuid.
continue
}
content, err := templateFS.ReadFile(filepath.Join("templates", name))
content, err := templateFS.ReadFile(templatePath(name))
if err != nil {
slog.Warn("bootstrap: failed to read embedded template", "file", name, "error", err)
continue
@@ -125,7 +122,7 @@ func SeedToStore(ctx context.Context, agentStore store.AgentStore, agentID uuid.
// Seed USER_PREDEFINED.md for predefined agents (agent-level, not in templateFiles).
// Provides baseline user-handling rules shared across all users.
if !hasContent[UserPredefinedFile] {
content, err := templateFS.ReadFile(filepath.Join("templates", UserPredefinedFile))
content, err := templateFS.ReadFile(templatePath(UserPredefinedFile))
if err == nil {
if err := retryOnBusy(func() error { return agentStore.SetAgentContextFile(ctx, agentID, UserPredefinedFile, string(content)) }); err != nil {
return seeded, err
@@ -218,7 +215,7 @@ func SeedUserFiles(ctx context.Context, agentStore store.AgentStore, agentID uui
if name == BootstrapFile || name == UserFile {
continue // skip bootstrap (the whole point) and user (already filled)
}
content, err := templateFS.ReadFile(filepath.Join("templates", name))
content, err := templateFS.ReadFile(templatePath(name))
if err != nil {
continue
}
@@ -286,7 +283,7 @@ func SeedUserFiles(ctx context.Context, agentStore store.AgentStore, agentID uui
templateName = "BOOTSTRAP_PREDEFINED.md"
}
content, err := templateFS.ReadFile(filepath.Join("templates", templateName))
content, err := templateFS.ReadFile(templatePath(templateName))
if err != nil {
slog.Warn("bootstrap: failed to read embedded template for user seed", "file", name, "error", err)
continue
@@ -319,7 +316,7 @@ func EmbeddedUserFiles(agentType string) []ContextFile {
if agentType == store.AgentTypePredefined && name == BootstrapFile {
templateName = "BOOTSTRAP_PREDEFINED.md"
}
content, err := templateFS.ReadFile(filepath.Join("templates", templateName))
content, err := templateFS.ReadFile(templatePath(templateName))
if err != nil {
continue
}
internal/channels/capabilities.go
+37
View File
@@ -0,0 +1,37 @@
package channels
import "errors"
// ErrMediaUnsupported is returned when a channel does not support media attachments.
// Callers (e.g. webhook handler) should either degrade to text-only or return HTTP 501.
var ErrMediaUnsupported = errors.New("channel does not support media attachments")
// mediaCapableTypes lists channel platform types that consume msg.Media in their Send()
// implementation. Verified against adapters:
// - telegram: internal/channels/telegram/send.go:251
// - discord: internal/channels/discord/discord.go:207
// - whatsapp: internal/channels/whatsapp/outbound.go:68
// - feishu: internal/channels/feishu/feishu.go:250
// - slack: internal/channels/slack/send.go:80
// - zalo_personal: internal/channels/zalo/personal/send.go:42
// - pancake: internal/channels/pancake/media_handler.go:18
// - facebook: internal/channels/facebook/facebook.go:205
//
// NOT in this list:
// - zalo_oa: internal/channels/zalo/zalo.go:115 — Send() does NOT consume msg.Media
var mediaCapableTypes = map[string]bool{
TypeTelegram: true,
TypeDiscord: true,
TypeWhatsApp: true,
TypeFeishu: true,
TypeSlack: true,
TypeZaloPersonal: true,
TypePancake: true,
TypeFacebook: true,
}
// IsMediaCapable reports whether the given channel platform type supports media attachments.
// Use Manager.ChannelTypeForName to resolve the type from a channel instance name.
func IsMediaCapable(channelType string) bool {
return mediaCapableTypes[channelType]
}
internal/channels/capabilities_test.go
+161
View File
@@ -0,0 +1,161 @@
package channels
import (
"context"
"errors"
"testing"
"github.com/nextlevelbuilder/goclaw/internal/bus"
)
// --- IsMediaCapable ---
func TestIsMediaCapable_KnownCapableTypes(t *testing.T) {
t.Parallel()
capable := []string{
TypeTelegram, TypeDiscord, TypeWhatsApp, TypeFeishu,
TypeSlack, TypeZaloPersonal, TypePancake, TypeFacebook,
}
for _, ct := range capable {
if !IsMediaCapable(ct) {
t.Errorf("IsMediaCapable(%q) = false, want true", ct)
}
}
}
func TestIsMediaCapable_UnsupportedTypes(t *testing.T) {
t.Parallel()
unsupported := []string{
TypeZaloOA, "unknown", "", "cli", "system",
}
for _, ct := range unsupported {
if IsMediaCapable(ct) {
t.Errorf("IsMediaCapable(%q) = true, want false", ct)
}
}
}
// --- SendMediaToChannel ---
// mockChannel implements Channel for testing SendMediaToChannel.
type mockChannel struct {
BaseChannel
channelType string
lastMsg bus.OutboundMessage
sendErr error
}
func newMockChannel(name, channelType string) *mockChannel {
mc := &mockChannel{channelType: channelType}
mc.BaseChannel = BaseChannel{name: name}
return mc
}
func (m *mockChannel) Type() string { return m.channelType }
func (m *mockChannel) Start(_ context.Context) error { return nil }
func (m *mockChannel) Stop(_ context.Context) error { return nil }
func (m *mockChannel) IsRunning() bool { return true }
func (m *mockChannel) IsAllowed(_ string) bool { return true }
func (m *mockChannel) Send(_ context.Context, msg bus.OutboundMessage) error {
m.lastMsg = msg
return m.sendErr
}
func TestSendMediaToChannel_PassesMediaToAdapter(t *testing.T) {
t.Parallel()
mb := bus.New()
mgr := NewManager(mb)
ch := newMockChannel("telegram-test", TypeTelegram)
mgr.channels["telegram-test"] = ch
media := []bus.MediaAttachment{
{URL: "/tmp/test.jpg", ContentType: "image/jpeg", Caption: "hello"},
}
err := mgr.SendMediaToChannel(context.Background(), "telegram-test", "chat123", "text", media)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(ch.lastMsg.Media) != 1 {
t.Fatalf("expected 1 media attachment, got %d", len(ch.lastMsg.Media))
}
if ch.lastMsg.Media[0].URL != "/tmp/test.jpg" {
t.Errorf("media URL mismatch: got %q", ch.lastMsg.Media[0].URL)
}
if ch.lastMsg.Content != "text" {
t.Errorf("content mismatch: got %q", ch.lastMsg.Content)
}
if ch.lastMsg.ChatID != "chat123" {
t.Errorf("chatID mismatch: got %q", ch.lastMsg.ChatID)
}
}
func TestSendMediaToChannel_ReturnsErrMediaUnsupported_ForZaloOA(t *testing.T) {
t.Parallel()
mb := bus.New()
mgr := NewManager(mb)
ch := newMockChannel("zalo-oa-test", TypeZaloOA)
mgr.channels["zalo-oa-test"] = ch
media := []bus.MediaAttachment{{URL: "/tmp/img.png", ContentType: "image/png"}}
err := mgr.SendMediaToChannel(context.Background(), "zalo-oa-test", "chat1", "", media)
if err == nil {
t.Fatal("expected error, got nil")
}
if !errors.Is(err, ErrMediaUnsupported) {
t.Errorf("expected ErrMediaUnsupported, got: %v", err)
}
}
func TestSendMediaToChannel_ErrorOnEmptyMedia(t *testing.T) {
t.Parallel()
mb := bus.New()
mgr := NewManager(mb)
ch := newMockChannel("telegram-test", TypeTelegram)
mgr.channels["telegram-test"] = ch
err := mgr.SendMediaToChannel(context.Background(), "telegram-test", "chat1", "text", nil)
if err == nil {
t.Fatal("expected error for empty media, got nil")
}
}
func TestSendMediaToChannel_ErrorOnChannelNotFound(t *testing.T) {
t.Parallel()
mb := bus.New()
mgr := NewManager(mb)
media := []bus.MediaAttachment{{URL: "/tmp/img.jpg", ContentType: "image/jpeg"}}
err := mgr.SendMediaToChannel(context.Background(), "nonexistent", "chat1", "", media)
if err == nil {
t.Fatal("expected error for unknown channel, got nil")
}
}
func TestSendToChannel_UnchangedByNewMethod(t *testing.T) {
t.Parallel()
mb := bus.New()
mgr := NewManager(mb)
ch := newMockChannel("telegram-test", TypeTelegram)
mgr.channels["telegram-test"] = ch
err := mgr.SendToChannel(context.Background(), "telegram-test", "chat1", "hello world")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if ch.lastMsg.Content != "hello world" {
t.Errorf("content mismatch: got %q", ch.lastMsg.Content)
}
if len(ch.lastMsg.Media) != 0 {
t.Errorf("expected no media, got %d attachments", len(ch.lastMsg.Media))
}
}
internal/channels/discord/handler.go
+25 -15
View File
@@ -44,12 +44,29 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate
peerKind = "direct"
}
// Pre-compute mention flag for groups so policy gating can suppress
// pairing replies when the bot was not addressed.
mentioned := false
if !isDM {
for _, u := range m.Mentions {
if u.ID == c.botUserID {
mentioned = true
break
}
}
if !mentioned && m.ReferencedMessage != nil &&
m.ReferencedMessage.Author != nil &&
m.ReferencedMessage.Author.ID == c.botUserID {
mentioned = true
}
}
if isDM {
if !c.checkDMPolicy(ctx, senderID, channelID) {
return
}
} else {
if !c.checkGroupPolicy(ctx, senderID, channelID) {
if !c.checkGroupPolicy(ctx, senderID, channelID, mentioned) {
slog.Debug("discord group message rejected by policy",
"user_id", senderID,
"username", senderName,
@@ -167,20 +184,8 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate
// Mention gating: in groups, only respond when bot is @mentioned (default true).
// When not mentioned, record message to pending history for later context.
// `mentioned` was pre-computed above for policy gating.
if peerKind == "group" && c.RequireMention() {
mentioned := false
for _, u := range m.Mentions {
if u.ID == c.botUserID {
mentioned = true
break
}
}
// Reply to bot's message counts as implicit mention.
if !mentioned && m.ReferencedMessage != nil &&
m.ReferencedMessage.Author != nil &&
m.ReferencedMessage.Author.ID == c.botUserID {
mentioned = true
}
if !mentioned {
// Collect media file paths for group history context.
var mediaPaths []string
@@ -318,12 +323,17 @@ func (c *Channel) handleMessage(_ *discordgo.Session, m *discordgo.MessageCreate
}
// checkGroupPolicy evaluates the group policy for a sender, with pairing support.
func (c *Channel) checkGroupPolicy(ctx context.Context, senderID, channelID string) bool {
// When RequireMention is enabled, pairing replies only fire if the bot was
// explicitly addressed — otherwise the bot stays silent in the channel.
func (c *Channel) checkGroupPolicy(ctx context.Context, senderID, channelID string, mentioned bool) bool {
result := c.CheckGroupPolicy(ctx, senderID, channelID, c.config.GroupPolicy)
switch result {
case channels.PolicyAllow:
return true
case channels.PolicyNeedsPairing:
if c.RequireMention() && !mentioned {
return false
}
groupSenderID := fmt.Sprintf("group:%s", channelID)
c.sendPairingReply(ctx, groupSenderID, channelID)
return false
internal/channels/dispatch.go
+30
View File
@@ -160,6 +160,36 @@ func (m *Manager) SendToChannel(ctx context.Context, channelName, chatID, conten
return channel.Send(ctx, msg)
}
// SendMediaToChannel delivers a message with media attachments to a specific channel by name.
// media must be non-empty; use SendToChannel for text-only messages.
// Returns ErrMediaUnsupported if the channel type does not support media.
func (m *Manager) SendMediaToChannel(ctx context.Context, channelName, chatID, content string, media []bus.MediaAttachment) error {
if len(media) == 0 {
return fmt.Errorf("SendMediaToChannel: media slice must not be empty; use SendToChannel for text-only messages")
}
m.mu.RLock()
channel, exists := m.channels[channelName]
m.mu.RUnlock()
if !exists {
return fmt.Errorf("channel %s not found", channelName)
}
if !IsMediaCapable(channel.Type()) {
return fmt.Errorf("%w: %s (%s)", ErrMediaUnsupported, channelName, channel.Type())
}
msg := bus.OutboundMessage{
Channel: channelName,
ChatID: chatID,
Content: content,
Media: media,
}
return channel.Send(ctx, msg)
}
// --- Send error notification helpers ---
// telegramAPIDescRe extracts the human-readable description from Telegram Bot API errors.
internal/channels/feishu/larkevents.go
+35 -10
View File
@@ -13,6 +13,8 @@ import (
"strings"
)
const maxWebhookBodyBytes = 1 << 20
// --- Event types (replacing larkim.P2MessageReceiveV1) ---
// MessageEvent is the parsed structure of a Feishu im.message.receive_v1 event.
@@ -42,9 +44,9 @@ type EventSender struct {
}
type EventMessage struct {
MessageID string `json:"message_id"`
RootID string `json:"root_id"`
ParentID string `json:"parent_id"`
MessageID string `json:"message_id"`
RootID string `json:"root_id"`
ParentID string `json:"parent_id"`
// ThreadID is the definitive "this message lives inside a thread" signal
// per Lark docs. Unlike RootID (which is populated on ANY reply — including
// plain quote replies), ThreadID is only present when the message is in an
@@ -59,8 +61,8 @@ type EventMessage struct {
}
type EventMention struct {
Key string `json:"key"`
ID struct {
Key string `json:"key"`
ID struct {
OpenID string `json:"open_id"`
UserID string `json:"user_id"`
UnionID string `json:"union_id"`
@@ -75,9 +77,9 @@ type EventMention struct {
// Schema v1.0 uses flat structure, v2.0 uses header+event.
type webhookEvent struct {
// v2.0 fields
Schema string `json:"schema"`
Header json.RawMessage `json:"header"`
Event json.RawMessage `json:"event"`
Schema string `json:"schema"`
Header json.RawMessage `json:"header"`
Event json.RawMessage `json:"event"`
// v1.0 fields (also used for URL verification challenge)
Type string `json:"type"`
@@ -97,11 +99,15 @@ func NewWebhookHandler(verificationToken, encryptKey string, onMessage func(even
return
}
body, err := io.ReadAll(r.Body)
body, err := io.ReadAll(io.LimitReader(r.Body, maxWebhookBodyBytes+1))
if err != nil {
http.Error(w, "read body failed", http.StatusBadRequest)
return
}
if len(body) > maxWebhookBodyBytes {
http.Error(w, "body too large", http.StatusRequestEntityTooLarge)
return
}
// Try to decrypt if encrypted
var envelope webhookEvent
@@ -129,11 +135,22 @@ func NewWebhookHandler(verificationToken, encryptKey string, onMessage func(even
// URL verification challenge
if envelope.Type == "url_verification" {
if verificationToken == "" || envelope.Token != verificationToken {
slog.Warn("security.feishu_webhook_url_verification_rejected")
w.WriteHeader(http.StatusOK)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(map[string]string{"challenge": envelope.Challenge})
return
}
if encryptKey != "" && envelope.Encrypt == "" {
slog.Warn("security.feishu_webhook_plaintext_rejected")
w.WriteHeader(http.StatusOK)
return
}
// Parse as message event
var event MessageEvent
@@ -144,8 +161,13 @@ func NewWebhookHandler(verificationToken, encryptKey string, onMessage func(even
}
// Verify token if configured
if verificationToken == "" && encryptKey == "" {
slog.Warn("security.feishu_webhook_missing_verification")
w.WriteHeader(http.StatusOK)
return
}
if verificationToken != "" && event.Header.Token != verificationToken {
slog.Warn("feishu webhook token mismatch")
slog.Warn("security.feishu_webhook_token_mismatch")
w.WriteHeader(http.StatusOK)
return
}
@@ -181,6 +203,9 @@ func decryptEvent(encryptedBase64, key string) ([]byte, error) {
// IV is first 16 bytes
iv := ciphertext[:aes.BlockSize]
ciphertext = ciphertext[aes.BlockSize:]
if len(ciphertext) == 0 || len(ciphertext)%aes.BlockSize != 0 {
return nil, fmt.Errorf("ciphertext length not block aligned")
}
mode := cipher.NewCBCDecrypter(block, iv)
mode.CryptBlocks(ciphertext, ciphertext)
internal/channels/feishu/larkevents_test.go
+104 -1
View File
@@ -63,7 +63,7 @@ func buildWebhookRequest(body string) *http.Request {
func TestWebhookHandler_URLVerification(t *testing.T) {
called := false
h := NewWebhookHandler("", "", func(_ *MessageEvent) { called = true })
h := NewWebhookHandler("test-tok", "", func(_ *MessageEvent) { called = true })
body := `{"type":"url_verification","token":"test-tok","challenge":"abc123"}`
w := httptest.NewRecorder()
@@ -84,6 +84,24 @@ func TestWebhookHandler_URLVerification(t *testing.T) {
}
}
func TestWebhookHandler_URLVerificationRequiresMatchingToken(t *testing.T) {
h := NewWebhookHandler("expected-token", "", func(_ *MessageEvent) {
t.Fatal("onMessage must not be called for url_verification")
})
body := `{"type":"url_verification","token":"wrong-token","challenge":"abc123"}`
w := httptest.NewRecorder()
h.ServeHTTP(w, buildWebhookRequest(body))
if w.Code != http.StatusOK {
t.Errorf("status: got %d, want 200", w.Code)
}
var resp map[string]string
if err := json.NewDecoder(w.Body).Decode(&resp); err == nil && resp["challenge"] != "" {
t.Fatalf("must not return challenge for mismatched token, got %q", resp["challenge"])
}
}
// --- Method not allowed ---
func TestWebhookHandler_MethodNotAllowed(t *testing.T) {
@@ -179,6 +197,72 @@ func TestWebhookHandler_TokenMatch_Dispatches(t *testing.T) {
}
}
func TestWebhookHandler_MissingVerificationTokenDoesNotDispatchMessage(t *testing.T) {
dispatched := make(chan *MessageEvent, 1)
h := NewWebhookHandler("", "", func(e *MessageEvent) { dispatched <- e })
env := map[string]any{
"schema": "2.0",
"header": map[string]any{
"event_id": "evt_missing_token",
"event_type": "im.message.receive_v1",
"token": "",
"app_id": "cli_test",
"tenant_key": "test-tenant-1",
},
"event": map[string]any{
"sender": map[string]any{},
"message": map[string]any{"message_id": "om_1", "chat_id": "oc_1"},
},
}
body, _ := json.Marshal(env)
w := httptest.NewRecorder()
h.ServeHTTP(w, buildWebhookRequest(string(body)))
if w.Code != http.StatusOK {
t.Errorf("status: got %d, want 200", w.Code)
}
select {
case <-dispatched:
t.Fatal("onMessage must not be called when verification token is missing")
case <-time.After(100 * time.Millisecond):
}
}
func TestWebhookHandler_EncryptKeyRejectsPlaintextEvent(t *testing.T) {
dispatched := make(chan *MessageEvent, 1)
h := NewWebhookHandler("", "encrypt-key", func(e *MessageEvent) { dispatched <- e })
env := map[string]any{
"schema": "2.0",
"header": map[string]any{
"event_id": "evt_plaintext",
"event_type": "im.message.receive_v1",
"token": "",
"app_id": "cli_test",
"tenant_key": "test-tenant-1",
},
"event": map[string]any{
"sender": map[string]any{},
"message": map[string]any{"message_id": "om_1", "chat_id": "oc_1"},
},
}
body, _ := json.Marshal(env)
w := httptest.NewRecorder()
h.ServeHTTP(w, buildWebhookRequest(string(body)))
if w.Code != http.StatusOK {
t.Errorf("status: got %d, want 200", w.Code)
}
select {
case <-dispatched:
t.Fatal("onMessage must not be called for plaintext event when encrypt key is configured")
case <-time.After(100 * time.Millisecond):
}
}
// --- Non-message event type ---
func TestWebhookHandler_NonMessageEvent_Ignored(t *testing.T) {
@@ -217,6 +301,18 @@ func TestWebhookHandler_InvalidJSON(t *testing.T) {
}
}
func TestWebhookHandler_RejectsOversizedBody(t *testing.T) {
h := NewWebhookHandler("", "", func(_ *MessageEvent) {
t.Fatal("onMessage must not be called for oversized body")
})
w := httptest.NewRecorder()
h.ServeHTTP(w, buildWebhookRequest(strings.Repeat("x", maxWebhookBodyBytes+1)))
if w.Code != http.StatusRequestEntityTooLarge {
t.Errorf("status: got %d, want 413", w.Code)
}
}
// --- Encrypted event ---
func TestWebhookHandler_EncryptedEvent_Decrypted(t *testing.T) {
@@ -281,6 +377,13 @@ func TestDecryptEvent_InvalidBase64(t *testing.T) {
}
}
func TestDecryptEvent_RejectsNonBlockMultipleCiphertext(t *testing.T) {
payload := base64.StdEncoding.EncodeToString([]byte("12345678901234567"))
if _, err := decryptEvent(payload, "key"); err == nil {
t.Fatal("expected error for non-block-multiple ciphertext")
}
}
func TestDecryptEvent_TooShort(t *testing.T) {
// Valid base64 but shorter than AES block size (16 bytes)
short := base64.StdEncoding.EncodeToString([]byte("short"))
internal/channels/pancake/pancake.go
+1 -2
View File
@@ -150,7 +150,7 @@ func (ch *Channel) Start(ctx context.Context) error {
if ch.webhookSecret == "" {
slog.Warn("security.pancake_webhook_no_secret",
"page_id", ch.pageID,
"note", "webhook_secret not configured; incoming webhook requests will not be authenticated")
"note", "webhook_secret not configured; incoming webhook requests will be ignored until configured")
}
// Without HMAC, any actor reaching the webhook endpoint can trigger Pancake API calls.
@@ -370,4 +370,3 @@ func (ch *Channel) maxMessageLength() int {
return 2000
}
}
internal/channels/pancake/pancake_loop_regression_test.go
+10 -6
View File
@@ -44,9 +44,10 @@ func TestMessageHandlerSkipsRecentOutboundEchoWithHTMLFormatting(t *testing.T) {
func TestWebhookRouterSkipsNonInboxConversationEvents(t *testing.T) {
msgBus := bus.New()
target := &Channel{
BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil),
pageID: "page-123",
platform: "facebook",
BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil),
pageID: "page-123",
platform: "facebook",
webhookSecret: "test-secret",
}
router := &webhookRouter{
instances: map[string]*Channel{
@@ -74,6 +75,7 @@ func TestWebhookRouterSkipsNonInboxConversationEvents(t *testing.T) {
}`
req := httptest.NewRequest(http.MethodPost, "/channels/pancake/webhook", strings.NewReader(body))
signTestPancakeRequest(req, body, target.webhookSecret)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
@@ -92,9 +94,10 @@ func TestWebhookRouterSkipsNonInboxConversationEvents(t *testing.T) {
func TestWebhookRouterPrefersMessageSenderOverConversationSender(t *testing.T) {
msgBus := bus.New()
target := &Channel{
BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil),
pageID: "page-123",
platform: "facebook",
BaseChannel: channels.NewBaseChannel(channels.TypePancake, msgBus, nil),
pageID: "page-123",
platform: "facebook",
webhookSecret: "test-secret",
}
router := &webhookRouter{
instances: map[string]*Channel{
@@ -127,6 +130,7 @@ func TestWebhookRouterPrefersMessageSenderOverConversationSender(t *testing.T) {
}`
req := httptest.NewRequest(http.MethodPost, "/channels/pancake/webhook", strings.NewReader(body))
signTestPancakeRequest(req, body, target.webhookSecret)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
internal/channels/pancake/pancake_test.go
+100 -11
View File
@@ -3,6 +3,9 @@ package pancake
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
@@ -537,12 +540,18 @@ func buildWebhookBody(pageID, convID, convType, senderID, msgID, content, postID
pageID, conv, msgID, content)
}
func signTestPancakeRequest(req *http.Request, body, secret string) {
mac := hmac.New(sha256.New, []byte(secret))
mac.Write([]byte(body))
req.Header.Set("X-Pancake-Signature", "sha256="+hex.EncodeToString(mac.Sum(nil)))
}
// newTestRouter creates an isolated webhookRouter with a registered channel.
func newTestRouter(t *testing.T, cfg pancakeInstanceConfig) (*webhookRouter, *Channel, *bus.MessageBus) {
t.Helper()
msgBus := bus.New()
cfg.PageID = "page-test"
creds := pancakeCreds{APIKey: "k", PageAccessToken: "t"}
creds := pancakeCreds{APIKey: "k", PageAccessToken: "t", WebhookSecret: "test-secret"}
ch, err := New(cfg, creds, msgBus, nil)
if err != nil {
t.Fatalf("New: %v", err)
@@ -557,10 +566,12 @@ func newTestRouter(t *testing.T, cfg pancakeInstanceConfig) (*webhookRouter, *Ch
func TestWebhookRouterRoutesCommentEvent(t *testing.T) {
cfg := pancakeInstanceConfig{}
cfg.Features.CommentReply = true
router, _, msgBus := newTestRouter(t, cfg)
router, ch, msgBus := newTestRouter(t, cfg)
ch.webhookSecret = "test-secret"
body := buildWebhookBody("page-test", "conv-1", "COMMENT", "user-1", "msg-1", "hello", "")
req := httptest.NewRequest(http.MethodPost, webhookPath, strings.NewReader(body))
signTestPancakeRequest(req, body, ch.webhookSecret)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
@@ -590,7 +601,7 @@ func TestWebhookRouterRoutesWebhookPageID(t *testing.T) {
msgBus := bus.New()
cfg.PageID = "pancake-internal-id"
creds := pancakeCreds{APIKey: "k", PageAccessToken: "t"}
creds := pancakeCreds{APIKey: "k", PageAccessToken: "t", WebhookSecret: "test-secret"}
ch, err := New(cfg, creds, msgBus, nil)
if err != nil {
t.Fatalf("New: %v", err)
@@ -611,6 +622,7 @@ func TestWebhookRouterRoutesWebhookPageID(t *testing.T) {
// Webhook arrives with Facebook native page ID — must route to the channel.
body := buildWebhookBody("fb-native-id", "conv-1", "COMMENT", "user-1", "msg-1", "hello", "")
req := httptest.NewRequest(http.MethodPost, webhookPath, strings.NewReader(body))
signTestPancakeRequest(req, body, ch.webhookSecret)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
@@ -632,10 +644,12 @@ func TestWebhookRouterRoutesWebhookPageID(t *testing.T) {
func TestWebhookRouterRoutesInboxEvent(t *testing.T) {
cfg := pancakeInstanceConfig{}
cfg.Features.InboxReply = true
router, _, msgBus := newTestRouter(t, cfg)
router, ch, msgBus := newTestRouter(t, cfg)
ch.webhookSecret = "test-secret"
body := buildWebhookBody("page-test", "conv-1", "INBOX", "user-1", "msg-2", "inbox msg", "")
req := httptest.NewRequest(http.MethodPost, webhookPath, strings.NewReader(body))
signTestPancakeRequest(req, body, ch.webhookSecret)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
@@ -651,6 +665,77 @@ func TestWebhookRouterRoutesInboxEvent(t *testing.T) {
}
}
func TestWebhookRouterMissingSecretDoesNotDispatch(t *testing.T) {
cfg := pancakeInstanceConfig{}
cfg.Features.InboxReply = true
router, ch, msgBus := newTestRouter(t, cfg)
ch.webhookSecret = ""
body := buildWebhookBody("page-test", "conv-1", "INBOX", "user-1", "msg-2", "inbox msg", "")
req := httptest.NewRequest(http.MethodPost, webhookPath, strings.NewReader(body))
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected provider-safe 200, got %d", w.Code)
}
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
if _, ok := msgBus.ConsumeInbound(ctx); ok {
t.Fatal("expected no dispatch when webhook secret is missing")
}
}
func TestWebhookRouterSignatureMismatchDoesNotDispatch(t *testing.T) {
cfg := pancakeInstanceConfig{}
cfg.Features.InboxReply = true
router, ch, msgBus := newTestRouter(t, cfg)
body := buildWebhookBody("page-test", "conv-1", "INBOX", "user-1", "msg-2", "inbox msg", "")
req := httptest.NewRequest(http.MethodPost, webhookPath, strings.NewReader(body))
signTestPancakeRequest(req, body, "wrong-secret")
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("expected provider-safe 200, got %d", w.Code)
}
if ch.webhookSecret == "" {
t.Fatal("test setup error: expected configured webhook secret")
}
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
if _, ok := msgBus.ConsumeInbound(ctx); ok {
t.Fatal("expected no dispatch on signature mismatch")
}
}
func TestWebhookRouterDuplicateSignedBodyDoesNotDispatchTwice(t *testing.T) {
cfg := pancakeInstanceConfig{}
cfg.Features.InboxReply = true
router, ch, msgBus := newTestRouter(t, cfg)
body := buildWebhookBody("page-test", "conv-1", "INBOX", "user-1", "", "inbox msg", "")
for i := 0; i < 2; i++ {
req := httptest.NewRequest(http.MethodPost, webhookPath, strings.NewReader(body))
signTestPancakeRequest(req, body, ch.webhookSecret)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("request %d status = %d, want 200", i+1, w.Code)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
if _, ok := msgBus.ConsumeInbound(ctx); !ok {
t.Fatal("expected first signed webhook to dispatch")
}
if _, ok := msgBus.ConsumeInbound(ctx); ok {
t.Fatal("expected duplicate signed webhook body to be skipped")
}
}
func TestWebhookRouterSkipsUnknownType(t *testing.T) {
cfg := pancakeInstanceConfig{}
cfg.Features.CommentReply = true
@@ -675,10 +760,12 @@ func TestWebhookRouterSkipsUnknownType(t *testing.T) {
func TestWebhookRouterCommentNormalizesPostID(t *testing.T) {
cfg := pancakeInstanceConfig{}
cfg.Features.CommentReply = true
router, _, msgBus := newTestRouter(t, cfg)
router, ch, msgBus := newTestRouter(t, cfg)
ch.webhookSecret = "test-secret"
body := buildWebhookBody("page-test", "conv-1", "COMMENT", "user-1", "msg-4", "hello", "post-123")
req := httptest.NewRequest(http.MethodPost, webhookPath, strings.NewReader(body))
signTestPancakeRequest(req, body, ch.webhookSecret)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
@@ -697,9 +784,9 @@ func TestWebhookRouterCommentNormalizesPostID(t *testing.T) {
// multiCaptureTransport records multiple requests (for first-inbox tests).
type multiCaptureTransport struct {
reqs []*http.Request
reqs []*http.Request
bodies [][]byte
mu sync.Mutex
mu sync.Mutex
}
func (t *multiCaptureTransport) RoundTrip(req *http.Request) (*http.Response, error) {
@@ -726,7 +813,7 @@ func newChannelWithMultiCapture(t *testing.T, cfg pancakeInstanceConfig) (*Chann
transport := &multiCaptureTransport{}
msgBus := bus.New()
cfg.PageID = "page-123"
creds := pancakeCreds{APIKey: "k", PageAccessToken: "t"}
creds := pancakeCreds{APIKey: "k", PageAccessToken: "t", WebhookSecret: "test-secret"}
ch, err := New(cfg, creds, msgBus, nil)
if err != nil {
t.Fatalf("New: %v", err)
@@ -1054,7 +1141,7 @@ func TestCommentFlowEndToEnd(t *testing.T) {
transport := &multiCaptureTransport{}
msgBus := bus.New()
cfg.PageID = "page-e2e"
creds := pancakeCreds{APIKey: "k", PageAccessToken: "t"}
creds := pancakeCreds{APIKey: "k", PageAccessToken: "t", WebhookSecret: "test-secret"}
ch, err := New(cfg, creds, msgBus, nil)
if err != nil {
t.Fatalf("New: %v", err)
@@ -1067,6 +1154,7 @@ func TestCommentFlowEndToEnd(t *testing.T) {
// Step 1: POST comment webhook.
body := buildWebhookBody("page-e2e", "conv-e2e", "COMMENT", "user-e2e", "msg-e2e", "great product!", "")
req := httptest.NewRequest(http.MethodPost, webhookPath, strings.NewReader(body))
signTestPancakeRequest(req, body, ch.webhookSecret)
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
@@ -1088,8 +1176,8 @@ func TestCommentFlowEndToEnd(t *testing.T) {
// Step 4: Send outbound reply.
outMsg := bus.OutboundMessage{
ChatID: inMsg.ChatID,
Content: "thank you!",
ChatID: inMsg.ChatID,
Content: "thank you!",
Metadata: inMsg.Metadata,
}
if err := ch.Send(context.Background(), outMsg); err != nil {
@@ -1122,6 +1210,7 @@ func TestCommentFlowEndToEnd(t *testing.T) {
// Step 6: Second comment from same sender — stateless: another DM fires.
body2 := buildWebhookBody("page-e2e", "conv-e2e", "COMMENT", "user-e2e", "msg-e2e-2", "another comment", "")
req2 := httptest.NewRequest(http.MethodPost, webhookPath, strings.NewReader(body2))
signTestPancakeRequest(req2, body2, ch.webhookSecret)
w2 := httptest.NewRecorder()
router.ServeHTTP(w2, req2)
internal/channels/pancake/webhook_handler.go
+24 -10
View File
@@ -34,6 +34,11 @@ func verifyHMAC(body []byte, secret, signature string) bool {
return hmac.Equal(got, expected)
}
func webhookReplayKey(body []byte) string {
sum := sha256.Sum256(body)
return "webhook:" + hex.EncodeToString(sum[:])
}
// --- Global webhook router for multi-page support ---
// webhookRouter routes incoming Pancake webhook events to the correct channel instance by page_id.
@@ -176,16 +181,25 @@ func (r *webhookRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
// HMAC signature verification — skip if webhook_secret not configured.
if target.webhookSecret != "" {
sig := req.Header.Get("X-Pancake-Signature")
if !verifyHMAC(body, target.webhookSecret, sig) {
slog.Warn("security.pancake_webhook_signature_mismatch",
"page_id", pageID,
"remote_addr", req.RemoteAddr)
w.WriteHeader(http.StatusOK)
return
}
if target.webhookSecret == "" {
slog.Warn("security.pancake_webhook_missing_secret",
"page_id", pageID,
"remote_addr", req.RemoteAddr)
w.WriteHeader(http.StatusOK)
return
}
sig := req.Header.Get("X-Pancake-Signature")
if !verifyHMAC(body, target.webhookSecret, sig) {
slog.Warn("security.pancake_webhook_signature_mismatch",
"page_id", pageID,
"remote_addr", req.RemoteAddr)
w.WriteHeader(http.StatusOK)
return
}
if target.isDup(webhookReplayKey(body)) {
slog.Info("pancake: duplicate webhook skipped", "page_id", pageID)
w.WriteHeader(http.StatusOK)
return
}
// Build normalized MessagingData from actual Pancake payload.
internal/config/config.go
+32
View File
@@ -56,9 +56,41 @@ type Config struct {
Tailscale TailscaleConfig `json:"tailscale"`
Bindings []AgentBinding `json:"bindings,omitempty"`
Hooks HooksConfig `json:"hooks"`
Packages PackagesConfig `json:"packages"` // runtime package mgmt (GitHub updater)
mu sync.RWMutex
}
// PackagesConfig tunes the runtime package update flow (Phase 1: GitHub
// binaries). GitHubToken is RESERVED for Phase 2 (authenticated rate-limit
// bump); currently unwired.
//
// UpdatesCheckTTL controls how stale the updates cache can get before a
// GET /v1/packages/updates triggers a background refresh. Encoded as
// human-readable string (e.g. "1h", "30m") parsed via time.ParseDuration;
// empty string → default 1h.
//
// ScratchDir is the tmp workspace used by the update executor for download
// + extract + staging before atomic swap. Defaults to "{BinDir}/../tmp" when
// empty; operators MAY set explicitly to avoid symlink-resolution issues
// (red-team H6).
type PackagesConfig struct {
GitHubToken string `json:"github_token,omitempty"` // Phase 2 stub
UpdatesCheckTTL string `json:"updates_check_ttl,omitempty"` // e.g. "1h"
ScratchDir string `json:"scratch_dir,omitempty"` // abs path
}
// UpdatesCheckTTLDuration parses UpdatesCheckTTL returning 1h on empty/invalid.
func (p PackagesConfig) UpdatesCheckTTLDuration() time.Duration {
if p.UpdatesCheckTTL == "" {
return time.Hour
}
d, err := time.ParseDuration(p.UpdatesCheckTTL)
if err != nil || d <= 0 {
return time.Hour
}
return d
}
// HooksConfig tunes the script-hook runtime caps. All zero-valued fields fall
// back to the handlers package defaults (see handlers.NewScriptHandler).
//
internal/config/config_channels.go
+23 -1
View File
@@ -218,6 +218,18 @@ type ProvidersConfig struct {
Novita ProviderConfig `json:"novita"` // Novita AI (OpenAI-compatible endpoint)
BytePlus ProviderConfig `json:"byteplus"` // BytePlus ModelArk (Seed 2.0)
BytePlusCoding ProviderConfig `json:"byteplus_coding"` // BytePlus ModelArk Coding Plan
Vertex VertexConfig `json:"vertex"` // Google Cloud Vertex AI (OAuth2 service account + ADC)
}
// VertexConfig configures Google Cloud Vertex AI.
// Credentials precedence: APIKey (inline JSON) > CredentialsFile (path) > ADC (both empty).
// ProjectID and Region are required; Model optional (defaults to google/gemini-2.0-flash-001).
type VertexConfig struct {
APIKey string `json:"api_key,omitempty"` // service account JSON inline (secret — never persist in config.json)
CredentialsFile string `json:"credentials_file,omitempty"` // path to service account JSON file
ProjectID string `json:"project_id,omitempty"`
Region string `json:"region,omitempty"`
Model string `json:"model,omitempty"`
}
// OllamaConfig configures a local (or self-hosted) Ollama instance.
@@ -292,6 +304,9 @@ func (p *ProvidersConfig) APIBaseForType(providerType string) string {
return p.BytePlus.APIBase
case "byteplus_coding":
return p.BytePlusCoding.APIBase
case "vertex":
// Computed from project+region at registration time; no config-level static base.
return ""
default:
return ""
}
@@ -321,7 +336,8 @@ func (c *Config) HasAnyProvider() bool {
p.ACP.Binary != "" ||
p.Novita.APIKey != "" ||
p.BytePlus.APIKey != "" ||
p.BytePlusCoding.APIKey != ""
p.BytePlusCoding.APIKey != "" ||
(p.Vertex.ProjectID != "" && p.Vertex.Region != "")
}
// QuotaWindow defines request limits per time window. Zero means unlimited.
@@ -428,9 +444,15 @@ type ToolPolicySpec struct {
Deny []string `json:"deny,omitempty"`
AlsoAllow []string `json:"alsoAllow,omitempty"`
ByProvider map[string]*ToolPolicySpec `json:"byProvider,omitempty"`
Wait *WaitToolPolicy `json:"wait,omitempty"`
ToolCallPrefix string `json:"toolCallPrefix,omitempty"` // prefix to strip from model's tool call names before registry lookup
}
// WaitToolPolicy configures per-agent safety bounds for the wait tool.
type WaitToolPolicy struct {
MinMs int `json:"min_ms,omitempty"`
MaxMs int `json:"max_ms,omitempty"`
}
// SessionsConfig controls session behavior.
// Matching TS src/config/sessions/types.ts + src/config/types.base.ts.
internal/config/config_load.go
+58 -1
View File
@@ -5,6 +5,8 @@ import (
"encoding/json"
"fmt"
"log/slog"
"net"
"net/netip"
"os"
"path/filepath"
"strconv"
@@ -13,6 +15,54 @@ import (
"github.com/titanous/json5"
)
const GatewayAllowInsecureNoAuthEnv = "GOCLAW_ALLOW_INSECURE_NO_AUTH"
// GatewayNoAuthFallbackAllowed reports whether empty-token gateway auth may
// run in local/dev compatibility mode.
func GatewayNoAuthFallbackAllowed(g GatewayConfig) bool {
if strings.TrimSpace(g.Token) != "" {
return false
}
if insecureNoAuthOptIn() {
return true
}
return isLoopbackGatewayHost(g.Host)
}
// ValidateGatewayAuth fails configurations that would expose the gateway
// without any bearer token.
func ValidateGatewayAuth(g GatewayConfig) error {
if strings.TrimSpace(g.Token) != "" || GatewayNoAuthFallbackAllowed(g) {
return nil
}
return fmt.Errorf("gateway token is required when GOCLAW_HOST=%q; set GOCLAW_GATEWAY_TOKEN or explicit %s=1 for local development only", g.Host, GatewayAllowInsecureNoAuthEnv)
}
func insecureNoAuthOptIn() bool {
switch strings.ToLower(strings.TrimSpace(os.Getenv(GatewayAllowInsecureNoAuthEnv))) {
case "1", "true", "yes", "on":
return true
default:
return false
}
}
func isLoopbackGatewayHost(host string) bool {
host = strings.TrimSpace(host)
if host == "" {
return false
}
if h, _, err := net.SplitHostPort(host); err == nil {
host = h
}
host = strings.Trim(host, "[]")
if strings.EqualFold(host, "localhost") {
return true
}
addr, err := netip.ParseAddr(host)
return err == nil && addr.IsLoopback()
}
// Default returns a Config with sensible defaults.
func Default() *Config {
return &Config{
@@ -109,6 +159,14 @@ func (c *Config) applyEnvOverrides() {
envStr("GOCLAW_OLLAMA_HOST", &c.Providers.Ollama.Host)
envStr("GOCLAW_OLLAMA_CLOUD_API_KEY", &c.Providers.OllamaCloud.APIKey)
envStr("GOCLAW_OLLAMA_CLOUD_API_BASE", &c.Providers.OllamaCloud.APIBase)
// Google Cloud Vertex AI (OAuth2 service account + ADC).
// APIKey may hold inline SA JSON; CredentialsFile is a path to SA JSON.
// If both empty, ADC (GOOGLE_APPLICATION_CREDENTIALS / gcloud / GCE metadata) is used.
envStr("GOCLAW_VERTEX_API_KEY", &c.Providers.Vertex.APIKey)
envStr("GOCLAW_VERTEX_CREDENTIALS_FILE", &c.Providers.Vertex.CredentialsFile)
envStr("GOCLAW_VERTEX_PROJECT_ID", &c.Providers.Vertex.ProjectID)
envStr("GOCLAW_VERTEX_REGION", &c.Providers.Vertex.Region)
envStr("GOCLAW_VERTEX_MODEL", &c.Providers.Vertex.Model)
envStr("GOCLAW_GATEWAY_TOKEN", &c.Gateway.Token)
envStr("GOCLAW_TELEGRAM_TOKEN", &c.Channels.Telegram.Token)
envStr("GOCLAW_DISCORD_TOKEN", &c.Channels.Discord.Token)
@@ -277,7 +335,6 @@ func (c *Config) applyEnvOverrides() {
}
}
// Save writes the config to a JSON file.
func Save(path string, cfg *Config) error {
cfg.mu.RLock()
internal/config/config_load_test.go
+33
View File
@@ -116,6 +116,39 @@ func TestLoad_EnvVarOverrides_InvalidPort(t *testing.T) {
}
}
func TestValidateGatewayAuthRejectsExternalNoToken(t *testing.T) {
cfg := Default()
cfg.Gateway.Host = "0.0.0.0"
cfg.Gateway.Token = ""
t.Setenv(GatewayAllowInsecureNoAuthEnv, "")
if err := ValidateGatewayAuth(cfg.Gateway); err == nil {
t.Fatal("expected external bind with empty gateway token to fail")
}
}
func TestValidateGatewayAuthAllowsLoopbackNoToken(t *testing.T) {
cfg := Default()
cfg.Gateway.Host = "127.0.0.1"
cfg.Gateway.Token = ""
t.Setenv(GatewayAllowInsecureNoAuthEnv, "")
if err := ValidateGatewayAuth(cfg.Gateway); err != nil {
t.Fatalf("loopback no-token mode should be allowed: %v", err)
}
}
func TestValidateGatewayAuthAllowsExplicitInsecureOptIn(t *testing.T) {
cfg := Default()
cfg.Gateway.Host = "0.0.0.0"
cfg.Gateway.Token = ""
t.Setenv(GatewayAllowInsecureNoAuthEnv, "1")
if err := ValidateGatewayAuth(cfg.Gateway); err != nil {
t.Fatalf("explicit insecure opt-in should allow no-token mode: %v", err)
}
}
// --- Env var for API keys ---
func TestLoad_EnvVarAPIKeys(t *testing.T) {
internal/config/config_secrets.go
+3
View File
@@ -37,6 +37,7 @@ func (c *Config) MaskedCopy() *Config {
maskNonEmpty(&cp.Providers.Zai.APIKey)
maskNonEmpty(&cp.Providers.ZaiCoding.APIKey)
maskNonEmpty(&cp.Providers.OllamaCloud.APIKey)
maskNonEmpty(&cp.Providers.Vertex.APIKey)
// Mask gateway token
maskNonEmpty(&cp.Gateway.Token)
@@ -84,6 +85,7 @@ func (c *Config) StripSecrets() {
c.Providers.Zai.APIKey = ""
c.Providers.ZaiCoding.APIKey = ""
c.Providers.OllamaCloud.APIKey = ""
c.Providers.Vertex.APIKey = ""
// Gateway token
c.Gateway.Token = ""
@@ -136,6 +138,7 @@ func (c *Config) StripMaskedSecrets() {
stripIfMasked(&c.Providers.Zai.APIKey)
stripIfMasked(&c.Providers.ZaiCoding.APIKey)
stripIfMasked(&c.Providers.OllamaCloud.APIKey)
stripIfMasked(&c.Providers.Vertex.APIKey)
// Gateway token
stripIfMasked(&c.Gateway.Token)
internal/crypto/env_denylist.go
+141
View File
@@ -0,0 +1,141 @@
// Package crypto — env_denylist.go provides env-key validation for grant env overrides.
// Reusable across HTTP handlers and any future validation layer.
package crypto
import (
"fmt"
"regexp"
"sort"
"strings"
)
// validEnvKeyShape is the regex for accepted env key shapes.
// Accepts uppercase letters, digits, and underscores only, starting with a letter or underscore.
// Rejects: lowercase, spaces, parentheses (Shellshock-class), empty.
var validEnvKeyShape = regexp.MustCompile(`^[A-Z_][A-Z0-9_]*$`)
// deniedExact is the exhaustive set of env keys that are rejected (case-insensitive, stored uppercase).
// Keep in sync with ENV_DENYLIST_EXACT in ui/web/src/pages/cli-credentials/cli-credential-grant-env-section.tsx.
var deniedExact = map[string]struct{}{
"PATH": {},
"HOME": {},
"USER": {},
"SHELL": {},
"PWD": {},
"LD_PRELOAD": {},
"LD_LIBRARY_PATH": {},
"LD_AUDIT": {},
"NODE_OPTIONS": {},
"NODE_PATH": {},
"PYTHONPATH": {},
"PYTHONHOME": {},
"PYTHONSTARTUP": {},
"GIT_SSH_COMMAND": {},
"GIT_SSH": {},
"GIT_EXEC_PATH": {},
"GIT_CONFIG_SYSTEM": {},
"SSH_AUTH_SOCK": {},
// Finding #6: additional dangerous vars for shell injection / TLS bypass / exfil
"BASH_ENV": {}, // sourced by non-interactive bash
"ENV": {}, // sourced by sh (non-interactive)
"PROMPT_COMMAND": {}, // executed before each shell prompt
"PERL5LIB": {}, // Perl library path override
"RUBYOPT": {}, // Ruby interpreter options
"HTTPS_PROXY": {}, // HTTPS exfiltration channel
"HTTP_PROXY": {}, // HTTP exfiltration channel
"NO_PROXY": {}, // disables proxy bypass
"SSL_CERT_FILE": {}, // TLS CA cert override — MitM
"SSL_CERT_DIR": {}, // TLS CA cert dir override — MitM
"CURL_CA_BUNDLE": {}, // curl TLS CA bundle override — MitM
"IFS": {}, // Internal Field Separator — shell injection
}
// deniedPrefixes is the set of uppercase key prefixes that are rejected.
// Keep in sync with ENV_DENYLIST_PREFIXES in ui/web/src/pages/cli-credentials/cli-credential-grant-env-section.tsx.
var deniedPrefixes = []string{
"DYLD_",
"GOCLAW_",
"LD_",
"NPM_CONFIG_", // npm lifecycle overrides (rc-style, loads modules); case-insensitive match via ToUpper
}
// maxGrantEnvKeys is the maximum number of env keys allowed per grant.
const maxGrantEnvKeys = 50
// maxGrantEnvValueBytes is the maximum byte length for a single env value.
const maxGrantEnvValueBytes = 4096
// IsDeniedEnvKey reports whether key is on the grant env denylist.
// Comparison is case-insensitive.
func IsDeniedEnvKey(key string) bool {
upper := strings.ToUpper(key)
if _, ok := deniedExact[upper]; ok {
return true
}
for _, pfx := range deniedPrefixes {
if strings.HasPrefix(upper, pfx) {
return true
}
}
return false
}
// ValidateGrantEnvVars checks all keys and values in envVars against the denylist
// and value constraints.
//
// Returns rejectedKeys (non-nil when any key is denied) and valueErr (first value violation).
// Callers should check rejectedKeys before valueErr.
//
// Rules:
// - Key count ≤ maxGrantEnvKeys
// - Key not on denylist (case-insensitive)
// - Value: no NUL byte, no newline, max maxGrantEnvValueBytes bytes
func ValidateGrantEnvVars(envVars map[string]string) (rejectedKeys []string, valueErr error) {
if len(envVars) > maxGrantEnvKeys {
return nil, fmt.Errorf("too many env keys: max %d, got %d", maxGrantEnvKeys, len(envVars))
}
// Finding #6: reject keys that don't match the valid key shape.
// This catches Shellshock-class injections (keys with `()`, whitespace, lowercase).
// Also catches empty key "".
// Finding #7: sort keys before iterating to produce deterministic error messages.
// Map iteration in Go is non-deterministic — without sorting, the same input can
// produce different error output on repeated calls, which is confusing for users.
keys := make([]string, 0, len(envVars))
for k := range envVars {
keys = append(keys, k)
}
sort.Strings(keys)
var denied []string
for _, k := range keys {
v := envVars[k]
// Key-shape validation: must match ^[A-Z_][A-Z0-9_]*$ (uppercase, no special chars).
if !validEnvKeyShape.MatchString(strings.ToUpper(k)) || k == "" {
return nil, fmt.Errorf("env key %q has invalid shape: must match ^[A-Z_][A-Z0-9_]*$ (uppercase, no spaces or special chars)", k)
}
if IsDeniedEnvKey(k) {
denied = append(denied, k)
}
if err := validateGrantEnvValue(v); err != nil {
return nil, fmt.Errorf("key %q: %w", k, err)
}
}
return denied, nil
}
func validateGrantEnvValue(v string) error {
if len(v) > maxGrantEnvValueBytes {
return fmt.Errorf("env value exceeds %d bytes", maxGrantEnvValueBytes)
}
for _, c := range v {
if c == 0 {
return fmt.Errorf("env value must not contain NUL bytes")
}
if c == '\n' || c == '\r' {
return fmt.Errorf("env value must not contain newlines")
}
}
return nil
}
internal/edition/edition.go
+17 -5
View File
@@ -18,17 +18,21 @@ type Edition struct {
RBACEnabled bool `json:"rbac_enabled"`
TeamFullMode bool `json:"team_full_mode"` // false = lite task actions only
VectorSearch bool `json:"vector_search"` // false = FTS5 only
SupportsPipNpm bool `json:"supports_pip_npm"` // false for Lite desktop
SupportsApk bool `json:"supports_apk"` // false for Lite desktop (no apk on macOS/Windows)
}
// --- Presets ---
// Standard is the default edition: all features enabled, no limits.
var Standard = Edition{
Name: "standard",
KGEnabled: true,
RBACEnabled: true,
TeamFullMode: true,
VectorSearch: true,
Name: "standard",
KGEnabled: true,
RBACEnabled: true,
TeamFullMode: true,
VectorSearch: true,
SupportsPipNpm: true,
SupportsApk: true,
}
// Lite is the desktop/self-hosted edition with sensible limits.
@@ -44,6 +48,8 @@ var Lite = Edition{
RBACEnabled: false,
TeamFullMode: false,
VectorSearch: false,
SupportsPipNpm: false,
SupportsApk: false,
}
// --- Global state ---
@@ -81,3 +87,9 @@ func (e Edition) ChannelLimit(channelType string) int {
}
return e.MaxChannels[channelType]
}
// AllowsChannels reports whether this edition permits channel-based webhook routes
// (kind="message"). Standard edition allows channels; Lite does not.
func (e Edition) AllowsChannels() bool {
return e.Name == "standard"
}
internal/edition/edition_test.go
+41
View File
@@ -366,6 +366,47 @@ func TestEditionConcurrentSafety(t *testing.T) {
// If this completes without panic, the test passes
}
// TestSupportsPipNpm verifies the pip/npm feature flag is set correctly per edition.
func TestSupportsPipNpm(t *testing.T) {
if !Standard.SupportsPipNpm {
t.Error("Standard.SupportsPipNpm = false, want true")
}
if Lite.SupportsPipNpm {
t.Error("Lite.SupportsPipNpm = true, want false")
}
}
// TestSupportsApk verifies the apk feature flag is set correctly per edition.
// Mirrors TestSupportsPipNpm pattern.
func TestSupportsApk(t *testing.T) {
if !Standard.SupportsApk {
t.Error("Standard.SupportsApk = false, want true")
}
if Lite.SupportsApk {
t.Error("Lite.SupportsApk = true, want false")
}
}
// TestEditionPresets_ApkField is a drift-guard that asserts BOTH presets
// explicitly spell out SupportsApk rather than relying on Go's zero-value.
// If someone removes the explicit line from either preset, this test catches
// the regression. (Red-team H-2 fix.)
func TestEditionPresets_ApkField(t *testing.T) {
// Standard must have SupportsApk = true (not zero-value false).
if !Standard.SupportsApk {
t.Error("Standard preset must explicitly set SupportsApk = true (drift guard: zero-value false would silently disable apk on Standard)")
}
// Lite must have SupportsApk = false (explicitly set, not just zero-value).
// We verify intent via the documented constraint: Lite.SupportsPipNpm must
// also be false, confirming the preset explicitly opts out of package managers.
if Lite.SupportsApk {
t.Error("Lite preset must have SupportsApk = false (apk unavailable on macOS/Windows desktop)")
}
if Lite.SupportsPipNpm {
t.Error("Lite preset must have SupportsPipNpm = false (package managers disabled on Lite)")
}
}
// TestCustomEdition_PartialConfiguration allows custom editions.
func TestCustomEdition_PartialConfiguration(t *testing.T) {
custom := Edition{
internal/eventbus/event_types.go
+6
View File
@@ -27,6 +27,12 @@ const (
EventDelegateCompleted EventType = "delegate.completed"
EventDelegateFailed EventType = "delegate.failed"
// Workstation lifecycle events (triggers BackendCache invalidation).
EventWorkstationUpdated EventType = "workstation.updated"
EventWorkstationDeleted EventType = "workstation.deleted"
// EventWorkstationPermChanged triggers AllowlistChecker cache invalidation (Phase 6).
// SourceID = workstation UUID.
EventWorkstationPermChanged EventType = "workstation.perm.changed"
)
// DomainEvent is a typed event with metadata for the consolidation pipeline.
internal/gateway/event_filter.go
+7
View File
@@ -131,6 +131,13 @@ func clientCanReceiveEvent(c *Client, event bus.Event) bool {
return true
}
// Package update events → only Owner clients (TenantID=Nil filter above).
// red-team B1/C5: explicit branch provides defense-in-depth even though the
// Admin/Owner path at line 46 already covers uuid.Nil events for owners.
if strings.HasPrefix(event.Name, "package.update.") {
return true
}
// Default: deny unknown events to non-admin (fail-closed).
return false
}
internal/gateway/methods/config_permissions.go
+63 -27
View File
@@ -38,6 +38,7 @@ func (m *ConfigPermissionsMethods) SetMemberResolver(r channels.MemberResolver)
func (m *ConfigPermissionsMethods) Register(router *gateway.MethodRouter) {
router.Register(protocol.MethodConfigPermissionsList, m.handleList)
router.Register(protocol.MethodConfigPermissionsCheck, m.handleCheck)
router.Register(protocol.MethodConfigPermissionsGrant, m.handleGrant)
router.Register(protocol.MethodConfigPermissionsRevoke, m.handleRevoke)
}
@@ -55,6 +56,10 @@ func (m *ConfigPermissionsMethods) handleList(ctx context.Context, client *gatew
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "agentId")))
return
}
if params.ConfigType != "" && !store.ValidConfigType(params.ConfigType) {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid configType"))
return
}
agentUUID, err := resolveAgentUUIDCached(ctx, m.agentRouter, m.agentStore, params.AgentID)
if err != nil {
@@ -71,6 +76,38 @@ func (m *ConfigPermissionsMethods) handleList(ctx context.Context, client *gatew
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"permissions": perms}))
}
func (m *ConfigPermissionsMethods) handleCheck(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
var params struct {
AgentID string `json:"agentId"`
Scope string `json:"scope"`
ConfigType string `json:"configType"`
UserID string `json:"userId"`
}
if req.Params != nil {
json.Unmarshal(req.Params, &params)
}
if errMsg := validateConfigPermissionParams(locale, params.AgentID, params.Scope, params.ConfigType, params.UserID, "allow", false); errMsg != "" {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, errMsg))
return
}
agentUUID, err := resolveAgentUUIDCached(ctx, m.agentRouter, m.agentStore, params.AgentID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid agentId"))
return
}
decision, err := store.CheckConfigPermissionDecision(ctx, m.permStore, agentUUID, params.Scope, params.ConfigType, params.UserID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal, configPermInternalErr("check", err)))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"decision": decision}))
}
func (m *ConfigPermissionsMethods) handleGrant(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
var params struct {
@@ -86,21 +123,8 @@ func (m *ConfigPermissionsMethods) handleGrant(ctx context.Context, client *gate
json.Unmarshal(req.Params, &params)
}
switch {
case params.AgentID == "":
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "agentId")))
return
case params.Scope == "":
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "scope")))
return
case params.ConfigType == "":
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "configType")))
return
case params.UserID == "":
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "userId")))
return
case params.Permission == "":
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "permission")))
if errMsg := validateConfigPermissionParams(locale, params.AgentID, params.Scope, params.ConfigType, params.UserID, params.Permission, true); errMsg != "" {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, errMsg))
return
}
@@ -158,18 +182,8 @@ func (m *ConfigPermissionsMethods) handleRevoke(ctx context.Context, client *gat
json.Unmarshal(req.Params, &params)
}
switch {
case params.AgentID == "":
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "agentId")))
return
case params.Scope == "":
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "scope")))
return
case params.ConfigType == "":
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "configType")))
return
case params.UserID == "":
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgRequired, "userId")))
if errMsg := validateConfigPermissionParams(locale, params.AgentID, params.Scope, params.ConfigType, params.UserID, "allow", false); errMsg != "" {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, errMsg))
return
}
@@ -191,3 +205,25 @@ func configPermInternalErr(action string, err error) string {
slog.Error("config.permissions RPC error", "action", action, "error", err)
return "internal error"
}
func validateConfigPermissionParams(locale, agentID, scope, configType, userID, permission string, validatePermission bool) string {
switch {
case agentID == "":
return i18n.T(locale, i18n.MsgRequired, "agentId")
case scope == "":
return i18n.T(locale, i18n.MsgRequired, "scope")
case configType == "":
return i18n.T(locale, i18n.MsgRequired, "configType")
case userID == "":
return i18n.T(locale, i18n.MsgRequired, "userId")
case !store.ValidConfigScope(scope):
return "invalid scope"
case !store.ValidConfigType(configType):
return "invalid configType"
case validatePermission && permission == "":
return i18n.T(locale, i18n.MsgRequired, "permission")
case validatePermission && !store.ValidConfigPermission(permission):
return "invalid permission"
}
return ""
}
internal/gateway/methods/skills.go
+59 -4
View File
@@ -4,19 +4,21 @@ import (
"context"
"encoding/json"
"log/slog"
"os"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/gateway"
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/internal/skills"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
// skillOwnerGetter is an optional interface for stores that can return a skill's owner ID.
type skillOwnerGetter interface {
GetSkillOwnerID(id uuid.UUID) (string, bool)
GetSkillOwnerID(ctx context.Context, id uuid.UUID) (string, bool)
}
// SkillsMethods handles skills.list, skills.get, skills.update.
@@ -38,6 +40,12 @@ func (m *SkillsMethods) Register(router *gateway.MethodRouter) {
func (m *SkillsMethods) handleList(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
allSkills := m.store.ListSkills(ctx)
// Visibility filter: non-admins see system skills, public skills, and
// their own private skills. Admins see everything in the tenant.
if !permissions.HasMinRole(client.Role(), permissions.RoleAdmin) {
allSkills = store.FilterVisibleSkills(ctx, allSkills)
}
result := make([]map[string]any, 0, len(allSkills))
for _, s := range allSkills {
entry := map[string]any{
@@ -64,6 +72,12 @@ func (m *SkillsMethods) handleList(ctx context.Context, client *gateway.Client,
if s.Author != "" {
entry["author"] = s.Author
}
if s.CreatorAgent != nil {
entry["creator_agent"] = s.CreatorAgent
}
if len(s.ManagerAgents) > 0 {
entry["manager_agents"] = s.ManagerAgents
}
if len(s.MissingDeps) > 0 {
entry["missing_deps"] = s.MissingDeps
}
@@ -116,7 +130,19 @@ func (m *SkillsMethods) handleGet(ctx context.Context, client *gateway.Client, r
return
}
content, _ := m.store.LoadSkill(ctx, params.Name)
// Visibility gate: hide private skills from non-owners (admins bypass).
if !permissions.HasMinRole(client.Role(), permissions.RoleAdmin) &&
!store.IsSkillVisibleTo(ctx, info.OwnerID, info.Visibility, info.IsSystem) {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrNotFound, i18n.T(locale, i18n.MsgNotFound, "skill", params.Name)))
return
}
content, ok := m.store.LoadSkill(ctx, info.Slug)
if !ok && info.Path != "" {
if b, err := os.ReadFile(info.Path); err == nil {
content = string(b)
}
}
resp := map[string]any{
"name": info.Name,
@@ -125,6 +151,7 @@ func (m *SkillsMethods) handleGet(ctx context.Context, client *gateway.Client, r
"source": info.Source,
"content": content,
"version": info.Version,
"enabled": info.Enabled,
}
if info.ID != "" {
resp["id"] = info.ID
@@ -135,6 +162,21 @@ func (m *SkillsMethods) handleGet(ctx context.Context, client *gateway.Client, r
if len(info.Tags) > 0 {
resp["tags"] = info.Tags
}
if info.Status != "" {
resp["status"] = info.Status
}
if info.Author != "" {
resp["author"] = info.Author
}
if info.CreatorAgent != nil {
resp["creator_agent"] = info.CreatorAgent
}
if len(info.ManagerAgents) > 0 {
resp["manager_agents"] = info.ManagerAgents
}
if len(info.MissingDeps) > 0 {
resp["missing_deps"] = info.MissingDeps
}
client.SendResponse(protocol.NewOKResponse(req.ID, resp))
}
@@ -196,20 +238,33 @@ func (m *SkillsMethods) handleUpdate(ctx context.Context, client *gateway.Client
return
}
// Ownership check: only skill owner or admin can update.
// Ownership check first: only skill owner or admin can update.
// Fail-closed: if store doesn't implement skillOwnerGetter, deny non-admin callers.
// Auth-before-validate avoids leaking skill-existence info via validation errors.
if !permissions.HasMinRole(client.Role(), permissions.RoleAdmin) {
ownerGetter, ok := m.store.(skillOwnerGetter)
if !ok {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrUnauthorized, i18n.T(locale, i18n.MsgPermissionDenied, "skills.update")))
return
}
if ownerID, found := ownerGetter.GetSkillOwnerID(skillID); found && ownerID != client.UserID() {
if ownerID, found := ownerGetter.GetSkillOwnerID(ctx, skillID); found && ownerID != client.UserID() {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrUnauthorized, i18n.T(locale, i18n.MsgPermissionDenied, "skills.update")))
return
}
}
// Validate visibility enum if present — fail closed before mutating the DB.
if v, ok := params.Updates["visibility"]; ok {
vs, _ := v.(string)
if err := skills.ValidateVisibility(vs); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgInvalidVisibility, vs)))
return
}
if vs != "" {
params.Updates["visibility"] = skills.NormalizeVisibility(vs)
}
}
if err := updater.UpdateSkill(ctx, skillID, params.Updates); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal, err.Error()))
return
internal/gateway/methods/workstations.go
+569
View File
@@ -0,0 +1,569 @@
package methods
import (
"context"
"database/sql"
"encoding/json"
"errors"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/gateway"
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/workstation"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
// WorkstationsMethods handles workstations.* RPC methods over WebSocket.
// Routes are only registered when !edition.IsLite() — callers must gate at registration.
type WorkstationsMethods struct {
wsStore store.WorkstationStore
linkStore store.AgentWorkstationLinkStore
permStore store.WorkstationPermissionStore // may be nil if Phase 6 not wired
activityStore store.WorkstationActivityStore // may be nil if Phase 7 not wired
}
// NewWorkstationsMethods creates WorkstationsMethods with the given stores.
func NewWorkstationsMethods(wsStore store.WorkstationStore, linkStore store.AgentWorkstationLinkStore) *WorkstationsMethods {
return &WorkstationsMethods{wsStore: wsStore, linkStore: linkStore}
}
// SetPermStore wires the permission store for allowlist CRUD methods.
func (m *WorkstationsMethods) SetPermStore(ps store.WorkstationPermissionStore) {
m.permStore = ps
}
// SetActivityStore wires the activity store for audit log methods (Phase 7).
func (m *WorkstationsMethods) SetActivityStore(as store.WorkstationActivityStore) {
m.activityStore = as
}
// Register wires the workstations.* methods onto the router.
// MUST only be called when edition is Standard (caller enforces the gate).
func (m *WorkstationsMethods) Register(router *gateway.MethodRouter) {
router.Register(protocol.MethodWorkstationsList, m.adminOnly(m.handleList))
router.Register(protocol.MethodWorkstationsGet, m.adminOnly(m.handleGet))
router.Register(protocol.MethodWorkstationsCreate, m.adminOnly(m.handleCreate))
router.Register(protocol.MethodWorkstationsUpdate, m.adminOnly(m.handleUpdate))
router.Register(protocol.MethodWorkstationsDelete, m.adminOnly(m.handleDelete))
router.Register(protocol.MethodWorkstationsTest, m.adminOnly(m.handleTestConnection))
router.Register(protocol.MethodWorkstationsLinkAgent, m.adminOnly(m.handleLinkAgent))
router.Register(protocol.MethodWorkstationsUnlinkAgent, m.adminOnly(m.handleUnlinkAgent))
// Phase 6: permission allowlist CRUD
router.Register(protocol.MethodWorkstationsPermList, m.adminOnly(m.handlePermList))
router.Register(protocol.MethodWorkstationsPermAdd, m.adminOnly(m.handlePermAdd))
router.Register(protocol.MethodWorkstationsPermRemove, m.adminOnly(m.handlePermRemove))
router.Register(protocol.MethodWorkstationsPermToggle, m.adminOnly(m.handlePermToggle))
// Phase 7: activity audit log
router.Register(protocol.MethodWorkstationsListActivity, m.adminOnly(m.handleListActivity))
}
// adminOnly is a middleware that requires at least RoleAdmin on the WS client.
func (m *WorkstationsMethods) adminOnly(next gateway.MethodHandler) gateway.MethodHandler {
return func(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
if !permissions.HasMinRole(client.Role(), permissions.RoleAdmin) {
locale := store.LocaleFromContext(ctx)
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrUnauthorized,
i18n.T(locale, i18n.MsgPermissionDenied, req.Method)))
return
}
next(ctx, client, req)
}
}
func (m *WorkstationsMethods) handleList(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
wss, err := m.wsStore.List(ctx)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgFailedToList, "workstations")))
return
}
views := make([]*store.SanitizedWorkstation, len(wss))
for i := range wss {
views[i] = wss[i].SanitizedView()
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"workstations": views}))
}
func (m *WorkstationsMethods) handleGet(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
var params struct {
ID string `json:"id"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
id, err := uuid.Parse(params.ID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "workstation")))
return
}
ws, err := m.wsStore.GetByID(ctx, id)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrNotFound,
i18n.T(locale, i18n.MsgWorkstationNotFound, params.ID)))
return
}
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgInternalError, err.Error())))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"workstation": ws.SanitizedView()}))
}
func (m *WorkstationsMethods) handleCreate(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
var params struct {
WorkstationKey string `json:"workstationKey"`
Name string `json:"name"`
BackendType store.WorkstationBackend `json:"backendType"`
Metadata json.RawMessage `json:"metadata"`
DefaultCWD string `json:"defaultCwd"`
DefaultEnv json.RawMessage `json:"defaultEnv"`
CreatedBy string `json:"createdBy"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
if params.WorkstationKey == "" {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgRequired, "workstationKey")))
return
}
if !workstation.ValidateWorkstationKey(params.WorkstationKey) {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidSlug, "workstationKey")))
return
}
if !workstation.ValidateBackend(params.BackendType) {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidBackend, string(params.BackendType))))
return
}
metaBytes := []byte(params.Metadata)
if err := store.ValidateMetadata(params.BackendType, metaBytes); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidMetadataShape, string(params.BackendType), err.Error())))
return
}
envBytes := []byte(params.DefaultEnv)
if len(envBytes) == 0 {
envBytes = []byte("{}")
}
ws := &store.Workstation{
WorkstationKey: params.WorkstationKey,
Name: params.Name,
BackendType: params.BackendType,
Metadata: metaBytes,
DefaultCWD: params.DefaultCWD,
DefaultEnv: envBytes,
Active: true,
CreatedBy: client.UserID(),
}
if err := m.wsStore.Create(ctx, ws); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgFailedToCreate, "workstation", err.Error())))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"workstation": ws.SanitizedView()}))
}
func (m *WorkstationsMethods) handleUpdate(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
var params struct {
ID string `json:"id"`
Updates map[string]any `json:"updates"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
id, err := uuid.Parse(params.ID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "workstation")))
return
}
if len(params.Updates) == 0 {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgNoUpdatesProvided)))
return
}
// I2 fix: validate metadata shape when metadata is being updated.
// Fetch current workstation to obtain backend_type for validation.
if _, hasMetadata := params.Updates["metadata"]; hasMetadata {
current, err := m.wsStore.GetByID(ctx, id)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrNotFound,
i18n.T(locale, i18n.MsgWorkstationNotFound, params.ID)))
return
}
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgInternalError, err.Error())))
return
}
metaBytes, err := json.Marshal(params.Updates["metadata"])
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidMetadataShape, string(current.BackendType), err.Error())))
return
}
if err := store.ValidateMetadata(current.BackendType, metaBytes); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidMetadataShape, string(current.BackendType), err.Error())))
return
}
}
if err := m.wsStore.Update(ctx, id, params.Updates); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgFailedToUpdate, "workstation", err.Error())))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"id": id}))
}
func (m *WorkstationsMethods) handleDelete(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
var params struct {
ID string `json:"id"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
id, err := uuid.Parse(params.ID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "workstation")))
return
}
if err := m.wsStore.Delete(ctx, id); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgFailedToDelete, "workstation", err.Error())))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"id": id}))
}
// handleTestConnection is a stub — real implementation in Phase 2/3.
func (m *WorkstationsMethods) handleTestConnection(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrNotImplemented,
i18n.T(locale, i18n.MsgNotImplemented, "workstations.testConnection")))
}
func (m *WorkstationsMethods) handleLinkAgent(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
var params struct {
AgentID string `json:"agentId"`
WorkstationID string `json:"workstationId"`
IsDefault bool `json:"isDefault"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
agentID, err := uuid.Parse(params.AgentID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "agent")))
return
}
wsID, err := uuid.Parse(params.WorkstationID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "workstation")))
return
}
link := &store.AgentWorkstationLink{
AgentID: agentID,
WorkstationID: wsID,
IsDefault: params.IsDefault,
}
if err := m.linkStore.Link(ctx, link); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgFailedToCreate, "agent_workstation_link", err.Error())))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"linked": true}))
}
func (m *WorkstationsMethods) handleUnlinkAgent(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
var params struct {
AgentID string `json:"agentId"`
WorkstationID string `json:"workstationId"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
agentID, err := uuid.Parse(params.AgentID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "agent")))
return
}
wsID, err := uuid.Parse(params.WorkstationID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "workstation")))
return
}
if err := m.linkStore.Unlink(ctx, agentID, wsID); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgFailedToDelete, "agent_workstation_link", err.Error())))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"unlinked": true}))
}
// --- Phase 6: workstation permission allowlist CRUD ---
func (m *WorkstationsMethods) requirePermStore(locale string, client *gateway.Client, req *protocol.RequestFrame) bool {
if m.permStore == nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrNotImplemented,
i18n.T(locale, i18n.MsgNotImplemented, "workstations.permissions")))
return false
}
return true
}
func (m *WorkstationsMethods) handlePermList(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
if !m.requirePermStore(locale, client, req) {
return
}
var params struct {
WorkstationID string `json:"workstationId"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
wsID, err := uuid.Parse(params.WorkstationID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "workstation")))
return
}
// Ownership check: verify workstation belongs to caller's tenant before listing perms.
// GetByID scopes the query by tenant_id — returns ErrNoRows for a different tenant.
if _, err := m.wsStore.GetByID(ctx, wsID); err != nil {
if errors.Is(err, sql.ErrNoRows) {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrNotFound,
i18n.T(locale, i18n.MsgWorkstationNotFound, params.WorkstationID)))
return
}
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgInternalError, err.Error())))
return
}
perms, err := m.permStore.ListForWorkstation(ctx, wsID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgFailedToList, "permissions")))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"permissions": perms}))
}
func (m *WorkstationsMethods) handlePermAdd(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
if !m.requirePermStore(locale, client, req) {
return
}
var params struct {
WorkstationID string `json:"workstationId"`
Pattern string `json:"pattern"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
wsID, err := uuid.Parse(params.WorkstationID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "workstation")))
return
}
// I5 fix: verify workstation belongs to caller's tenant before adding permission.
// GetByID scopes the query by tenant_id in the WHERE clause — returns ErrNoRows if
// the workstation exists in a different tenant.
if _, err := m.wsStore.GetByID(ctx, wsID); err != nil {
if errors.Is(err, sql.ErrNoRows) {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrNotFound,
i18n.T(locale, i18n.MsgWorkstationNotFound, params.WorkstationID)))
return
}
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgInternalError, err.Error())))
return
}
if params.Pattern == "" {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgRequired, "pattern")))
return
}
perm := &store.WorkstationPermission{
WorkstationID: wsID,
Pattern: params.Pattern,
Enabled: true,
CreatedBy: client.UserID(),
}
if err := m.permStore.Add(ctx, perm); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgFailedToCreate, "permission", err.Error())))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"permission": perm}))
}
func (m *WorkstationsMethods) handlePermRemove(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
if !m.requirePermStore(locale, client, req) {
return
}
var params struct {
ID string `json:"id"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
id, err := uuid.Parse(params.ID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "permission")))
return
}
if err := m.permStore.Remove(ctx, id); err != nil {
if errors.Is(err, sql.ErrNoRows) {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrNotFound,
i18n.T(locale, i18n.MsgWorkstationPermNotFound, params.ID)))
return
}
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgFailedToDelete, "permission", err.Error())))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"id": id}))
}
func (m *WorkstationsMethods) handlePermToggle(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
if !m.requirePermStore(locale, client, req) {
return
}
var params struct {
ID string `json:"id"`
Enabled bool `json:"enabled"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
id, err := uuid.Parse(params.ID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "permission")))
return
}
if err := m.permStore.SetEnabled(ctx, id, params.Enabled); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgFailedToUpdate, "permission", err.Error())))
return
}
client.SendResponse(protocol.NewOKResponse(req.ID, map[string]any{"id": id, "enabled": params.Enabled}))
}
// --- Phase 7: activity audit log ---
func (m *WorkstationsMethods) handleListActivity(ctx context.Context, client *gateway.Client, req *protocol.RequestFrame) {
locale := store.LocaleFromContext(ctx)
if m.activityStore == nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrNotImplemented,
i18n.T(locale, i18n.MsgNotImplemented, "workstations.activity.list")))
return
}
var params struct {
WorkstationID string `json:"workstationId"`
Limit int `json:"limit"`
Cursor string `json:"cursor"`
}
if req.Params != nil {
if err := json.Unmarshal(req.Params, &params); err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest, "invalid params"))
return
}
}
wsID, err := uuid.Parse(params.WorkstationID)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInvalidRequest,
i18n.T(locale, i18n.MsgInvalidID, "workstation")))
return
}
// Ownership check: verify the workstation belongs to the caller's tenant.
// GetByID scopes by tenant_id — returns ErrNoRows if workstation is in a different tenant.
if _, err := m.wsStore.GetByID(ctx, wsID); err != nil {
if errors.Is(err, sql.ErrNoRows) {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrNotFound,
i18n.T(locale, i18n.MsgWorkstationNotFound, params.WorkstationID)))
return
}
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgInternalError, err.Error())))
return
}
limit := params.Limit
if limit <= 0 || limit > 200 {
limit = 50
}
var cursor *uuid.UUID
if params.Cursor != "" {
if cID, err := uuid.Parse(params.Cursor); err == nil {
cursor = &cID
}
}
rows, nextCursor, err := m.activityStore.List(ctx, wsID, limit, cursor)
if err != nil {
client.SendResponse(protocol.NewErrorResponse(req.ID, protocol.ErrInternal,
i18n.T(locale, i18n.MsgFailedToList, "activity")))
return
}
resp := map[string]any{"activity": rows}
if nextCursor != nil {
resp["nextCursor"] = nextCursor.String()
}
client.SendResponse(protocol.NewOKResponse(req.ID, resp))
}
internal/gateway/router.go
+3 -2
View File
@@ -10,6 +10,7 @@ import (
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/cache"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/edition"
httpapi "github.com/nextlevelbuilder/goclaw/internal/http"
"github.com/nextlevelbuilder/goclaw/internal/i18n"
@@ -233,7 +234,7 @@ func (r *MethodRouter) handleConnect(ctx context.Context, client *Client, req *p
}
// Path 2: No token configured → operator (backward compat)
if configToken == "" {
if configToken == "" && config.GatewayNoAuthFallbackAllowed(r.server.cfg.Gateway) {
client.role = permissions.RoleOperator
client.authenticated = true
client.userID = params.UserID
@@ -260,7 +261,7 @@ func (r *MethodRouter) handleConnect(ctx context.Context, client *Client, req *p
if paired {
client.role = permissions.RoleOperator
client.authenticated = true
client.userID = params.UserID
client.userID = params.UserID
client.pairedSenderID = params.SenderID
client.pairedChannel = "browser"
tid, errCode := r.resolveTenantHint(ctx, params.TenantHint, params.UserID)
internal/gateway/router_test.go
+64
View File
@@ -0,0 +1,64 @@
package gateway
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
func TestHandleConnectRejectsNoTokenExternalBind(t *testing.T) {
cfg := config.Default()
cfg.Gateway.Host = "0.0.0.0"
cfg.Gateway.Token = ""
t.Setenv(config.GatewayAllowInsecureNoAuthEnv, "")
server := NewServer(cfg, nil, nil, nil)
client := NewClient(nil, server, "203.0.113.10")
req := &protocol.RequestFrame{ID: "req-1", Method: protocol.MethodConnect}
server.router.Handle(context.Background(), client, req)
if client.authenticated {
t.Fatal("expected unauthenticated client for external no-token connect")
}
if client.role != "" {
t.Fatalf("role = %q, want empty", client.role)
}
select {
case raw := <-client.send:
var resp protocol.ResponseFrame
if err := json.Unmarshal(raw, &resp); err != nil {
t.Fatalf("unmarshal response: %v", err)
}
if resp.Error == nil || resp.Error.Code != protocol.ErrUnauthorized {
t.Fatalf("response error = %#v, want unauthorized", resp.Error)
}
case <-time.After(500 * time.Millisecond):
t.Fatal("expected unauthorized response")
}
}
func TestHandleConnectAllowsExplicitInsecureNoTokenOptIn(t *testing.T) {
cfg := config.Default()
cfg.Gateway.Host = "0.0.0.0"
cfg.Gateway.Token = ""
t.Setenv(config.GatewayAllowInsecureNoAuthEnv, "1")
server := NewServer(cfg, nil, nil, nil)
client := NewClient(nil, server, "127.0.0.1")
req := &protocol.RequestFrame{ID: "req-1", Method: protocol.MethodConnect}
server.router.Handle(context.Background(), client, req)
if !client.authenticated {
t.Fatal("expected authenticated client with explicit insecure opt-in")
}
if client.role != permissions.RoleOperator {
t.Fatalf("role = %q, want operator", client.role)
}
}
internal/gateway/server.go
+37 -9
View File
@@ -21,11 +21,11 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/config"
httpapi "github.com/nextlevelbuilder/goclaw/internal/http"
mcpbridge "github.com/nextlevelbuilder/goclaw/internal/mcp"
"github.com/nextlevelbuilder/goclaw/internal/webui"
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/tools"
"github.com/nextlevelbuilder/goclaw/internal/webui"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
@@ -50,21 +50,21 @@ type Server struct {
// Non-handler dependencies (don't implement RegisterRoutes)
policyEngine *permissions.PolicyEngine
pairingService store.PairingStore
apiKeyStore store.APIKeyStore // for API key auth lookup
agentStore store.AgentStore // for context injection in tools_invoke
msgBus *bus.MessageBus // for MCP bridge media delivery
apiKeyStore store.APIKeyStore // for API key auth lookup
agentStore store.AgentStore // for context injection in tools_invoke
msgBus *bus.MessageBus // for MCP bridge media delivery
upgrader websocket.Upgrader
rateLimiter *RateLimiter
clients map[string]*Client
mu sync.RWMutex
startedAt time.Time
version string
db interface{ PingContext(context.Context) error } // for health check DB ping
updateChecker *UpdateChecker
startedAt time.Time
version string
db interface{ PingContext(context.Context) error } // for health check DB ping
updateChecker *UpdateChecker
logTee *LogTee // optional; auto-unsubscribes clients on disconnect
logTee *LogTee // optional; auto-unsubscribes clients on disconnect
postTurn tools.PostTurnProcessor // optional; for team task dispatch in HTTP API paths
httpServer *http.Server
@@ -464,6 +464,11 @@ func (s *Server) SetPackagesHandler(h *httpapi.PackagesHandler) {
s.handlers = append(s.handlers, h)
}
// SetGatewayUpgradeHandler sets the host-local gateway upgrade trigger handler.
func (s *Server) SetGatewayUpgradeHandler(h *httpapi.GatewayUpgradeHandler) {
s.handlers = append(s.handlers, h)
}
// SetOAuthHandler sets the OAuth handler (available in all modes).
func (s *Server) SetOAuthHandler(h *httpapi.OAuthHandler) { s.handlers = append(s.handlers, h) }
@@ -472,6 +477,24 @@ func (s *Server) SetAPIKeysHandler(h *httpapi.APIKeysHandler) {
s.handlers = append(s.handlers, h)
}
// SetWebhooksAdminHandler registers the webhook admin CRUD handler.
func (s *Server) SetWebhooksAdminHandler(h *httpapi.WebhooksAdminHandler) {
s.handlers = append(s.handlers, h)
}
// SetWebhookMessageHandler registers the POST /v1/webhooks/message runtime handler.
// Only called when edition.Current().AllowsChannels() is true (Standard edition).
func (s *Server) SetWebhookMessageHandler(h *httpapi.WebhookMessageHandler) {
s.handlers = append(s.handlers, h)
}
// SetWebhookLLMHandler registers the POST /v1/webhooks/llm runtime handler.
// Available in all editions (Standard + Lite). Localhost-only enforcement is
// handled by WebhookAuthMiddleware at request time via webhook.LocalhostOnly.
func (s *Server) SetWebhookLLMHandler(h *httpapi.WebhookLLMHandler) {
s.handlers = append(s.handlers, h)
}
// SetTenantsHandler sets the tenant management handler.
func (s *Server) SetTenantsHandler(h *httpapi.TenantsHandler) {
s.handlers = append(s.handlers, h)
@@ -576,6 +599,11 @@ func (s *Server) SetAgentStore(as store.AgentStore) { s.agentStore = as }
// SetMessageBus sets the message bus for MCP bridge media delivery.
func (s *Server) SetMessageBus(mb *bus.MessageBus) { s.msgBus = mb }
// SetWorkstationsHandler sets the workstations CRUD handler (Standard edition only).
func (s *Server) SetWorkstationsHandler(h *httpapi.WorkstationsHandler) {
s.handlers = append(s.handlers, h)
}
// SetVersion sets the server version for health responses.
func (s *Server) SetVersion(v string) { s.version = v }
internal/hooks/handlers/command.go
+16 -7
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"os"
"os/exec"
"runtime"
"github.com/nextlevelbuilder/goclaw/internal/edition"
"github.com/nextlevelbuilder/goclaw/internal/hooks"
@@ -52,13 +53,13 @@ func (h *CommandHandler) Execute(ctx context.Context, cfg hooks.HookConfig, ev h
return hooks.DecisionError, fmt.Errorf("hook: command handler: marshal event: %w", err)
}
sh, err := findShell()
sh, args, err := shellCommand(cmd)
if err != nil {
return hooks.DecisionError, fmt.Errorf("hook: command handler: %w", err)
}
//nolint:gosec // Command comes from admin-configured hooks stored in DB, not user input.
c := exec.CommandContext(ctx, sh, "-c", cmd)
c := exec.CommandContext(ctx, sh, args...)
c.Stdin = bytes.NewReader(eventJSON)
c.Env = buildAllowedEnv(allowedVars)
@@ -95,12 +96,20 @@ func (h *CommandHandler) Execute(ctx context.Context, cfg hooks.HookConfig, ev h
return hooks.DecisionAllow, nil
}
// findShell returns the path to sh, falling back to /bin/sh.
func findShell() (string, error) {
if p, err := exec.LookPath("sh"); err == nil {
return p, nil
// shellCommand returns the platform shell invocation for an admin-configured command.
func shellCommand(cmd string) (string, []string, error) {
if runtime.GOOS == "windows" {
for _, shell := range []string{"pwsh", "powershell"} {
if p, err := exec.LookPath(shell); err == nil {
return p, []string{"-NoProfile", "-NonInteractive", "-Command", cmd}, nil
}
}
return "", nil, fmt.Errorf("PowerShell shell not found")
}
return "/bin/sh", nil
if p, err := exec.LookPath("sh"); err == nil {
return p, []string{"-c", cmd}, nil
}
return "/bin/sh", []string{"-c", cmd}, nil
}
// buildAllowedEnv constructs an env slice containing only the listed keys from
internal/hooks/handlers/command_test.go
+11 -2
View File
@@ -2,6 +2,7 @@ package handlers_test
import (
"context"
"runtime"
"strings"
"testing"
"time"
@@ -60,7 +61,11 @@ func TestCommand_ExitOne_ReturnsError(t *testing.T) {
func TestCommand_JSONContinueFalse_ReturnsBlock(t *testing.T) {
h := &handlers.CommandHandler{Edition: edition.Lite}
// printf produces {"continue":false} on stdout then exits 0.
cfg := makeCmdCfg(`printf '{"continue":false}'`, hooks.ScopeAgent)
command := `printf '{"continue":false}'`
if runtime.GOOS == "windows" {
command = `Write-Output '{"continue":false}'`
}
cfg := makeCmdCfg(command, hooks.ScopeAgent)
dec, err := h.Execute(context.Background(), cfg, hooks.Event{HookEvent: hooks.EventPreToolUse})
if err != nil {
t.Fatalf("unexpected error: %v", err)
@@ -137,7 +142,11 @@ func TestCommand_CtxCancel_IsKilled(t *testing.T) {
// "exec sleep 30" replaces the shell process with sleep directly (no
// grandchild), so exec.CommandContext's kill reaches the process that
// holds the pipe — Output() returns promptly after ctx deadline fires.
cfg := makeCmdCfg("exec sleep 30", hooks.ScopeAgent)
command := "exec sleep 30"
if runtime.GOOS == "windows" {
command = "Start-Sleep -Seconds 30"
}
cfg := makeCmdCfg(command, hooks.ScopeAgent)
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
internal/hooks/handlers/http_test.go
+20 -9
View File
@@ -6,6 +6,7 @@ import (
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"github.com/nextlevelbuilder/goclaw/internal/crypto"
"github.com/nextlevelbuilder/goclaw/internal/hooks"
@@ -13,6 +14,16 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/security"
)
// testCtx returns a context with a 10s deadline for HTTP handler tests.
// This prevents tests from consuming the entire package timeout budget
// when the CI runner is slow.
func testCtx(t *testing.T) context.Context {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
t.Cleanup(cancel)
return ctx
}
// makeHTTPCfg builds a minimal HookConfig with given URL.
func makeHTTPCfg(url string) hooks.HookConfig {
return hooks.HookConfig{
@@ -34,7 +45,7 @@ func TestHTTP_200Allow(t *testing.T) {
defer srv.Close()
h := &handlers.HTTPHandler{Client: srv.Client()}
dec, err := h.Execute(context.Background(), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
dec, err := h.Execute(testCtx(t), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -55,7 +66,7 @@ func TestHTTP_200BlockDecision(t *testing.T) {
defer srv.Close()
h := &handlers.HTTPHandler{Client: srv.Client()}
dec, err := h.Execute(context.Background(), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
dec, err := h.Execute(testCtx(t), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -76,7 +87,7 @@ func TestHTTP_200ContinueFalse(t *testing.T) {
defer srv.Close()
h := &handlers.HTTPHandler{Client: srv.Client()}
dec, err := h.Execute(context.Background(), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
dec, err := h.Execute(testCtx(t), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -102,7 +113,7 @@ func TestHTTP_5xxRetriesOnce(t *testing.T) {
defer srv.Close()
h := &handlers.HTTPHandler{Client: srv.Client()}
dec, err := h.Execute(context.Background(), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
dec, err := h.Execute(testCtx(t), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
if err != nil {
t.Fatalf("unexpected error after retry: %v", err)
}
@@ -128,7 +139,7 @@ func TestHTTP_4xxReturnsError(t *testing.T) {
defer srv.Close()
h := &handlers.HTTPHandler{Client: srv.Client()}
dec, err := h.Execute(context.Background(), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
dec, err := h.Execute(testCtx(t), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
if err == nil {
t.Fatal("expected error on persistent 400")
}
@@ -149,7 +160,7 @@ func TestHTTP_MissingURL(t *testing.T) {
Config: map[string]any{}, // no "url"
Enabled: true,
}
dec, err := h.Execute(context.Background(), cfg, hooks.Event{HookEvent: hooks.EventPreToolUse})
dec, err := h.Execute(testCtx(t), cfg, hooks.Event{HookEvent: hooks.EventPreToolUse})
if err == nil {
t.Fatal("expected error for missing URL")
}
@@ -169,7 +180,7 @@ func TestHTTP_NonJSON2xx_TreatedAsAllow(t *testing.T) {
defer srv.Close()
h := &handlers.HTTPHandler{Client: srv.Client()}
dec, err := h.Execute(context.Background(), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
dec, err := h.Execute(testCtx(t), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@@ -216,7 +227,7 @@ func TestHTTP_EncryptedAuthHeader_Decrypted(t *testing.T) {
},
Enabled: true,
}
dec, err := h.Execute(context.Background(), cfg, hooks.Event{HookEvent: hooks.EventPreToolUse})
dec, err := h.Execute(testCtx(t), cfg, hooks.Event{HookEvent: hooks.EventPreToolUse})
if err != nil {
t.Fatalf("unexpected error: %v (got Authorization: %q)", err, gotAuth)
}
@@ -244,7 +255,7 @@ func TestHTTP_ResponseBodyCappedAt1MiB(t *testing.T) {
h := &handlers.HTTPHandler{Client: srv.Client()}
// 2 MiB body is non-JSON → treated as allow, no panic.
dec, err := h.Execute(context.Background(), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
dec, err := h.Execute(testCtx(t), makeHTTPCfg(srv.URL), hooks.Event{HookEvent: hooks.EventPreToolUse})
if err != nil {
t.Fatalf("unexpected error on oversized body: %v", err)
}
internal/http/agents.go
+38
View File
@@ -287,6 +287,10 @@ func (h *AgentsHandler) handleCreate(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgInvalidRequest, err.Error()))
return
}
if err := validateAgentModelFallback(req.ModelFallback); err != nil {
writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgInvalidRequest, err.Error()))
return
}
if err := h.agents.Create(r.Context(), &req); err != nil {
if strings.Contains(err.Error(), "duplicate key") || strings.Contains(err.Error(), "23505") {
@@ -451,6 +455,20 @@ func (h *AgentsHandler) handleUpdate(w http.ResponseWriter, r *http.Request) {
return
}
validationAgent.ChatGPTOAuthRouting = rawRouting
allowed["chatgpt_oauth_routing"] = rawRouting
}
if fallback, ok := allowed["model_fallback"]; ok {
rawFallback, err := marshalJSONRaw(fallback)
if err != nil {
writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgInvalidJSON))
return
}
if err := validateAgentModelFallback(rawFallback); err != nil {
writeError(w, http.StatusBadRequest, protocol.ErrInvalidRequest, i18n.T(locale, i18n.MsgInvalidRequest, err.Error()))
return
}
validationAgent.ModelFallback = rawFallback
allowed["model_fallback"] = rawFallback
}
if err := validateChatGPTOAuthAgentRouting(
@@ -497,6 +515,26 @@ func (h *AgentsHandler) handleUpdate(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]string{"ok": "true"})
}
func validateAgentModelFallback(raw json.RawMessage) error {
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var cfg store.ModelFallbackConfig
if err := json.Unmarshal(raw, &cfg); err != nil {
return fmt.Errorf("invalid model_fallback")
}
normalized := store.NormalizeModelFallbackConfig(&cfg)
if normalized == nil {
return nil
}
for _, candidate := range normalized.Candidates {
if candidate.Provider == "" || candidate.Model == "" {
return fmt.Errorf("fallback candidates require provider and model")
}
}
return nil
}
// syncIdentityName updates the Name: field in the agent's IDENTITY.md (agent-level and
// all per-user copies for open agents) so the agent self-reports the new display name.
// Errors are logged but do not fail the rename request.
internal/http/agents_export_marshal.go
+2
View File
@@ -124,6 +124,7 @@ func marshalAgentConfig(ag *store.AgentData) ([]byte, error) {
ReasoningConfig json.RawMessage `json:"reasoning_config,omitempty"`
WorkspaceSharing json.RawMessage `json:"workspace_sharing,omitempty"`
ChatGPTOAuthRouting json.RawMessage `json:"chatgpt_oauth_routing,omitempty"`
ModelFallback json.RawMessage `json:"model_fallback,omitempty"`
ShellDenyGroups json.RawMessage `json:"shell_deny_groups,omitempty"`
KGDedupConfig json.RawMessage `json:"kg_dedup_config,omitempty"`
}
@@ -154,6 +155,7 @@ func marshalAgentConfig(ag *store.AgentData) ([]byte, error) {
ReasoningConfig: ag.ReasoningConfig,
WorkspaceSharing: ag.WorkspaceSharing,
ChatGPTOAuthRouting: canonicalizeChatGPTOAuthRoutingForResponse(ag.ChatGPTOAuthRouting),
ModelFallback: ag.ModelFallback,
ShellDenyGroups: ag.ShellDenyGroups,
KGDedupConfig: ag.KGDedupConfig,
}, "", " ")
internal/http/agents_import_agent.go
+1
View File
@@ -111,6 +111,7 @@ func (h *AgentsHandler) buildAgentFromArchive(cfg map[string]json.RawMessage, ag
ag.ReasoningConfig = rawOrNil(cfg["reasoning_config"])
ag.WorkspaceSharing = rawOrNil(cfg["workspace_sharing"])
ag.ChatGPTOAuthRouting = rawOrNil(cfg["chatgpt_oauth_routing"])
ag.ModelFallback = rawOrNil(cfg["model_fallback"])
ag.ShellDenyGroups = rawOrNil(cfg["shell_deny_groups"])
ag.KGDedupConfig = rawOrNil(cfg["kg_dedup_config"])
internal/http/auth.go
+9 -2
View File
@@ -79,6 +79,7 @@ func extractAgentID(r *http.Request, model string) string {
// --- Package-level API key cache for shared auth ---
var pkgGatewayToken string
var pkgNoAuthFallbackAllowed = true
var pkgAPIKeyCache *apiKeyCache
var pkgPairingStore store.PairingStore
var pkgTenantCache *tenantCache
@@ -90,6 +91,12 @@ func InitGatewayToken(token string) {
pkgGatewayToken = token
}
// InitGatewayNoAuthFallbackAllowed controls the legacy empty-token local/dev
// fallback after startup config validation.
func InitGatewayNoAuthFallbackAllowed(allowed bool) {
pkgNoAuthFallbackAllowed = allowed
}
// InitAPIKeyCache initializes the shared API key cache with TTL and pubsub invalidation.
// Must be called once during server startup before handling requests.
func InitAPIKeyCache(s store.APIKeyStore, mb *bus.MessageBus) {
@@ -233,8 +240,8 @@ func resolveAuthWithBearer(r *http.Request, bearer string) authResult {
slog.Warn("security.http_pairing_auth_failed", "sender_id", senderID, "ip", r.RemoteAddr)
}
}
// No auth configured → admin (no token = dev/single-user mode, full access)
if pkgGatewayToken == "" {
// No auth configured → admin only when startup allowed local/dev fallback.
if pkgGatewayToken == "" && pkgNoAuthFallbackAllowed {
return authResult{Role: permissions.RoleAdmin, Authenticated: true, TenantID: store.MasterTenantID}
}
return authResult{}
internal/http/auth_test.go
+21
View File
@@ -36,6 +36,13 @@ func setupTestToken(t *testing.T, token string) {
t.Cleanup(func() { pkgGatewayToken = old })
}
func setupTestNoAuthFallback(t *testing.T, allowed bool) {
t.Helper()
old := pkgNoAuthFallbackAllowed
pkgNoAuthFallbackAllowed = allowed
t.Cleanup(func() { pkgNoAuthFallbackAllowed = old })
}
func setupTestTenantStore(t *testing.T, ts store.TenantStore) {
t.Helper()
old := pkgTenantCache
@@ -221,6 +228,7 @@ func TestResolveAuth_WrongToken(t *testing.T) {
func TestResolveAuth_NoAuthConfigured(t *testing.T) {
setupTestCache(t, nil)
setupTestNoAuthFallback(t, true)
r := httptest.NewRequest("GET", "/v1/agents", nil)
@@ -233,6 +241,19 @@ func TestResolveAuth_NoAuthConfigured(t *testing.T) {
}
}
func TestResolveAuth_NoAuthConfiguredDisallowed(t *testing.T) {
setupTestCache(t, nil)
setupTestToken(t, "")
setupTestNoAuthFallback(t, false)
r := httptest.NewRequest("GET", "/v1/agents", nil)
auth := resolveAuth(r)
if auth.Authenticated {
t.Fatal("expected unauthenticated when no-token fallback is disabled")
}
}
func TestResolveAuth_APIKeyReadScope(t *testing.T) {
// We need to hash the token the same way crypto.HashAPIKey does
// For testing, we'll inject directly into the cache
internal/http/file_token.go
+14 -9
View File
@@ -6,7 +6,7 @@ import (
"crypto/sha256"
"encoding/base64"
"fmt"
"path/filepath"
"path"
"regexp"
"strconv"
"strings"
@@ -77,21 +77,26 @@ func SignMediaPath(rawPath, secret string) string {
return ""
}
// Strip stale ?ft= tokens
path := staleTokenRe.ReplaceAllString(rawPath, "")
path = strings.TrimRight(path, "?&")
cleanPath := filepathToURLPath(rawPath)
cleanPath = staleTokenRe.ReplaceAllString(cleanPath, "")
cleanPath = strings.TrimRight(cleanPath, "?&")
// Strip all /v1/files/ and /v1/media/ prefixes (may be stacked from legacy bugs)
for strings.Contains(path, "/v1/files/") {
path = strings.Replace(path, "/v1/files/", "/", 1)
for strings.Contains(cleanPath, "/v1/files/") {
cleanPath = strings.Replace(cleanPath, "/v1/files/", "/", 1)
}
for strings.Contains(path, "/v1/media/") {
path = strings.Replace(path, "/v1/media/", "/", 1)
for strings.Contains(cleanPath, "/v1/media/") {
cleanPath = strings.Replace(cleanPath, "/v1/media/", "/", 1)
}
path = filepath.Clean(path)
urlPath := "/v1/files/" + strings.TrimPrefix(path, "/")
cleanPath = path.Clean(cleanPath)
urlPath := "/v1/files/" + strings.TrimPrefix(cleanPath, "/")
ft := SignFileToken(urlPath, secret, FileTokenTTL)
return urlPath + "?ft=" + ft
}
func filepathToURLPath(rawPath string) string {
return strings.ReplaceAll(rawPath, `\`, "/")
}
// fileURLRe matches /v1/files/... and /v1/media/... URLs in markdown and plain text.
// Captures the full URL path (stops at whitespace, closing paren, quote, or angle bracket).
var fileURLRe = regexp.MustCompile(`(/v1/(?:files|media)/[^\s)"'<>]+)`)
internal/http/files.go
+165 -91
View File
@@ -25,6 +25,8 @@ type FilesHandler struct {
dataDir string // data directory root for tenant path validation
}
var filesAfterOpenHookForTest func(string)
// NewFilesHandler creates a handler that serves files by absolute path.
// workspace is the root directory used for fallback generated file search.
// dataDir is used for tenant path validation (files must be within tenant's dirs).
@@ -54,39 +56,15 @@ func (h *FilesHandler) handleSign(w http.ResponseWriter, r *http.Request) {
return
}
// Validate path is within workspace or dataDir before signing.
// Defense-in-depth: prevents signing tokens for arbitrary system files.
absPath := filepath.Clean(body.Path)
if !filepath.IsAbs(absPath) {
// Windows drive letter path (e.g. "C:\...") — keep as-is, consistent with handleServe.
if len(absPath) >= 2 && absPath[1] == ':' {
// already absolute on Windows
} else {
absPath = filepath.Clean("/" + absPath)
}
}
sep := string(filepath.Separator)
if (h.workspace == "" || (!strings.HasPrefix(absPath, h.workspace+sep) && absPath != h.workspace)) &&
(h.dataDir == "" || (!strings.HasPrefix(absPath, h.dataDir+sep) && absPath != h.dataDir)) {
slog.Warn("security.files_sign_path_denied", "path", absPath, "workspace", h.workspace, "data_dir", h.dataDir)
absPath := absoluteFilePath(body.Path)
file, _, _, ok := h.openValidatedFile(authedReq, absPath, false)
if !ok {
http.Error(w, `{"error":"path outside allowed directories"}`, http.StatusForbidden)
return
}
_ = file.Close()
// Multi-tenant (RBAC): additionally restrict to the requesting tenant's dirs.
// Prevents tenant A from signing a URL for tenant B's files.
if edition.Current().RBACEnabled {
tenantData := config.TenantDataDir(h.dataDir, store.TenantIDFromContext(authedReq.Context()), store.TenantSlugFromContext(authedReq.Context()))
tenantWs := config.TenantWorkspace(h.workspace, store.TenantIDFromContext(authedReq.Context()), store.TenantSlugFromContext(authedReq.Context()))
if (!strings.HasPrefix(absPath, tenantData+sep) && absPath != tenantData) &&
(!strings.HasPrefix(absPath, tenantWs+sep) && absPath != tenantWs) {
slog.Warn("security.files_sign_tenant_denied", "path", absPath, "tenant_data", tenantData, "tenant_ws", tenantWs)
http.Error(w, `{"error":"path outside allowed directories"}`, http.StatusForbidden)
return
}
}
urlPath := "/v1/files/" + strings.TrimPrefix(filepath.Clean(body.Path), "/")
urlPath := fileURLPath(absPath)
ft := SignFileToken(urlPath, FileSigningKey(), FileTokenTTL)
writeJSON(w, http.StatusOK, map[string]string{
"url": urlPath + "?ft=" + ft,
@@ -139,69 +117,20 @@ func (h *FilesHandler) handleServe(w http.ResponseWriter, r *http.Request) {
return
}
// URL path is the absolute path with leading "/" stripped (e.g. "app/.goclaw/workspace/file.png")
// Windows drive letter: "C:/Users/..." → use directly without prepending "/"
var absPath string
if len(urlPath) >= 2 && urlPath[1] == ':' {
absPath = filepath.Clean(urlPath)
} else {
absPath = filepath.Clean("/" + urlPath)
}
absPath := absoluteFilePath(urlPath)
// Block access to sensitive system directories
for _, prefix := range deniedFilePrefixes {
if strings.HasPrefix(absPath, prefix) {
slog.Warn("security.files_denied_path", "path", absPath)
http.Error(w, i18n.T(locale, i18n.MsgInvalidPath), http.StatusForbidden)
return
}
if hasDeniedFilePrefix(absPath) {
slog.Warn("security.files_denied_path", "path", absPath)
http.Error(w, i18n.T(locale, i18n.MsgInvalidPath), http.StatusForbidden)
return
}
// Defense-in-depth: validate workspace/dataDir boundary even for signed file tokens.
// The token cryptographically binds the URL path, but we also verify the resolved
// absolute path stays within allowed directories to limit blast radius of any
// bug in the signing flow.
if r.URL.Query().Get("ft") != "" {
sep := string(filepath.Separator)
inWorkspace := h.workspace != "" && (strings.HasPrefix(absPath, h.workspace+sep) || absPath == h.workspace)
inDataDir := h.dataDir != "" && (strings.HasPrefix(absPath, h.dataDir+sep) || absPath == h.dataDir)
if !inWorkspace && !inDataDir {
slog.Warn("security.files_ft_path_denied", "path", absPath, "workspace", h.workspace, "data_dir", h.dataDir)
http.NotFound(w, r)
return
}
}
// Path isolation: validate file path is within allowed directories.
if r.URL.Query().Get("ft") == "" {
allowed := false
// Always allow files within workspace root and data dir root.
// These are the two top-level directories that contain all user files.
sep := string(filepath.Separator)
if h.workspace != "" && (strings.HasPrefix(absPath, h.workspace+sep) || absPath == h.workspace) {
allowed = true
}
if !allowed && h.dataDir != "" && (strings.HasPrefix(absPath, h.dataDir+sep) || absPath == h.dataDir) {
allowed = true
}
// Multi-tenant (standard edition): additionally restrict to tenant-scoped subdirectories.
if allowed && edition.Current().RBACEnabled {
tenantData := config.TenantDataDir(h.dataDir, store.TenantIDFromContext(r.Context()), store.TenantSlugFromContext(r.Context()))
tenantWs := h.tenantWorkspace(r)
if !strings.HasPrefix(absPath, tenantData+sep) &&
!strings.HasPrefix(absPath, tenantWs+sep) &&
absPath != tenantData && absPath != tenantWs {
allowed = false
}
}
if !allowed {
slog.Warn("security.files_path_denied", "path", absPath, "workspace", h.workspace, "data_dir", h.dataDir)
http.NotFound(w, r)
return
}
signed := r.URL.Query().Get("ft") != ""
if !h.lexicallyAllowsFilePath(r, absPath, signed) {
slog.Warn("security.files_path_denied", "path", absPath, "workspace", h.workspace, "data_dir", h.dataDir)
http.NotFound(w, r)
return
}
info, err := os.Stat(absPath)
@@ -221,7 +150,7 @@ func (h *FilesHandler) handleServe(w http.ResponseWriter, r *http.Request) {
// For ft= signed requests, the path is cryptographically bound — no fallback search.
// Searching the global workspace could cross tenant boundaries if a same-basename
// file exists in another tenant's directory.
if r.URL.Query().Get("ft") != "" {
if signed {
http.NotFound(w, r)
return
}
@@ -237,6 +166,13 @@ func (h *FilesHandler) handleServe(w http.ResponseWriter, r *http.Request) {
return
}
}
file, realPath, fileInfo, ok := h.openValidatedFile(r, absPath, signed)
if !ok {
http.NotFound(w, r)
return
}
defer file.Close()
absPath = realPath
// Set Content-Type from extension
ext := filepath.Ext(absPath)
@@ -250,7 +186,146 @@ func (h *FilesHandler) handleServe(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filepath.Base(absPath)))
}
http.ServeFile(w, r, absPath)
http.ServeContent(w, r, filepath.Base(absPath), fileInfo.ModTime(), file)
}
func absoluteFilePath(path string) string {
absPath := filepath.Clean(path)
if filepath.IsAbs(absPath) {
return absPath
}
// Windows drive letter path (e.g. "C:\...") is absolute to this handler.
if len(absPath) >= 2 && absPath[1] == ':' {
return absPath
}
return filepath.Clean(string(filepath.Separator) + absPath)
}
func fileURLPath(absPath string) string {
return "/v1/files/" + strings.TrimPrefix(filepath.Clean(absPath), string(filepath.Separator))
}
func hasDeniedFilePrefix(path string) bool {
cleaned := filepath.Clean(path)
for _, prefix := range deniedFilePrefixes {
root := filepath.Clean(prefix)
if pathWithinDir(cleaned, root) {
return true
}
}
return false
}
func configuredFileRoot(root string) string {
if root == "" {
return ""
}
return filepath.Clean(root)
}
func canonicalFileRoots(roots []string) []string {
out := make([]string, 0, len(roots))
for _, root := range roots {
if root = configuredFileRoot(root); root != "" {
out = append(out, evalSymlinkOrClean(root))
}
}
return out
}
func (h *FilesHandler) requestFileRoots(r *http.Request, signed bool, absPath string) []string {
if signed {
return []string{
inferredScopedFileRoot(h.workspace, absPath),
inferredScopedFileRoot(h.dataDir, absPath),
}
}
if edition.Current().RBACEnabled {
return []string{
config.TenantWorkspace(h.workspace, store.TenantIDFromContext(r.Context()), store.TenantSlugFromContext(r.Context())),
config.TenantDataDir(h.dataDir, store.TenantIDFromContext(r.Context()), store.TenantSlugFromContext(r.Context())),
}
}
return []string{h.workspace, h.dataDir}
}
func inferredScopedFileRoot(base, absPath string) string {
base = configuredFileRoot(base)
if base == "" || !pathWithinDir(filepath.Clean(absPath), base) {
return ""
}
tenantsRoot := filepath.Join(base, "tenants")
if !pathWithinDir(filepath.Clean(absPath), tenantsRoot) || filepath.Clean(absPath) == tenantsRoot {
return base
}
rel, err := filepath.Rel(tenantsRoot, filepath.Clean(absPath))
if err != nil {
return ""
}
first, _, _ := strings.Cut(rel, string(filepath.Separator))
if first == "" || first == "." || first == ".." {
return ""
}
return filepath.Join(tenantsRoot, first)
}
func filePathWithinAnyRoot(path string, roots []string) bool {
for _, root := range roots {
if root != "" && pathWithinDir(filepath.Clean(path), filepath.Clean(root)) {
return true
}
}
return false
}
func (h *FilesHandler) lexicallyAllowsFilePath(r *http.Request, absPath string, signed bool) bool {
return filePathWithinAnyRoot(absPath, h.requestFileRoots(r, signed, absPath))
}
func (h *FilesHandler) openValidatedFile(r *http.Request, absPath string, signed bool) (*os.File, string, os.FileInfo, bool) {
file, err := os.Open(absPath)
if err != nil {
return nil, "", nil, false
}
if filesAfterOpenHookForTest != nil {
filesAfterOpenHookForTest(absPath)
}
realPath, err := filepath.EvalSymlinks(absPath)
if err != nil {
slog.Warn("security.files_path_unresolved", "path", absPath, "error", err)
_ = file.Close()
return nil, "", nil, false
}
realPath = filepath.Clean(realPath)
if hasDeniedFilePrefix(realPath) {
slog.Warn("security.files_realpath_denied", "path", absPath, "resolved", realPath)
_ = file.Close()
return nil, "", nil, false
}
roots := canonicalFileRoots(h.requestFileRoots(r, signed, absPath))
if !filePathWithinAnyRoot(realPath, roots) {
slog.Warn("security.files_realpath_escape", "path", absPath, "resolved", realPath, "roots", roots)
_ = file.Close()
return nil, "", nil, false
}
realInfo, err := os.Stat(realPath)
if err != nil {
_ = file.Close()
return nil, "", nil, false
}
fileInfo, err := file.Stat()
if err != nil {
_ = file.Close()
slog.Warn("security.files_open_race", "path", realPath, "error", err)
return nil, "", nil, false
}
if fileInfo.IsDir() || realInfo.IsDir() || !os.SameFile(realInfo, fileInfo) {
_ = file.Close()
slog.Warn("security.files_open_race", "path", realPath)
return nil, "", nil, false
}
return file, realPath, fileInfo, true
}
// tenantWorkspace resolves the workspace scoped to the requesting tenant.
@@ -339,4 +414,3 @@ func isNumeric(s string) bool {
}
return len(s) > 0
}
internal/http/files_path_security_test.go
+89
View File
@@ -5,6 +5,7 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
)
@@ -141,6 +142,94 @@ func TestFilesHandleServe_FileOutsideAllDirs_WithToken_Returns404(t *testing.T)
}
}
func TestFilesHandleServe_SignedSymlinkEscape_Returns404(t *testing.T) {
h, workspace := makeTestFilesHandler(t)
outsideDir := t.TempDir()
target := filepath.Join(outsideDir, "secret.txt")
if err := os.WriteFile(target, []byte("secret"), 0644); err != nil {
t.Fatal(err)
}
linkPath := filepath.Join(workspace, "link.txt")
if err := os.Symlink(target, linkPath); err != nil {
t.Skipf("symlink unavailable: %v", err)
}
urlPath := "/v1/files/" + strings.TrimPrefix(filepath.Clean(linkPath), "/")
ft := SignFileToken(urlPath, FileSigningKey(), FileTokenTTL)
mux := http.NewServeMux()
mux.HandleFunc("GET /v1/files/{path...}", h.handleServe)
req := httptest.NewRequest(http.MethodGet, urlPath+"?ft="+ft, nil)
w := httptest.NewRecorder()
mux.ServeHTTP(w, req)
if w.Code == http.StatusOK {
t.Fatal("signed symlink escaping workspace should not be served")
}
}
func TestFilesHandleServe_OpenThenSwapToSymlinkEscape_Returns404(t *testing.T) {
h, workspace := makeTestFilesHandler(t)
outsideDir := t.TempDir()
secretPath := filepath.Join(outsideDir, "secret.txt")
if err := os.WriteFile(secretPath, []byte("secret"), 0644); err != nil {
t.Fatal(err)
}
filePath := filepath.Join(workspace, "race.txt")
if err := os.WriteFile(filePath, []byte("allowed"), 0644); err != nil {
t.Fatal(err)
}
filesAfterOpenHookForTest = func(opened string) {
if opened != filePath {
return
}
_ = os.Remove(filePath)
_ = os.Symlink(secretPath, filePath)
}
defer func() { filesAfterOpenHookForTest = nil }()
urlPath := "/v1/files/" + strings.TrimPrefix(filepath.Clean(filePath), "/")
ft := SignFileToken(urlPath, FileSigningKey(), FileTokenTTL)
mux := http.NewServeMux()
mux.HandleFunc("GET /v1/files/{path...}", h.handleServe)
req := httptest.NewRequest(http.MethodGet, urlPath+"?ft="+ft, nil)
w := httptest.NewRecorder()
mux.ServeHTTP(w, req)
if w.Code == http.StatusOK {
t.Fatal("file swapped to escaping symlink after open should not be served")
}
if strings.Contains(w.Body.String(), "secret") {
t.Fatal("response leaked swapped outside file content")
}
}
func TestFilesHandleSign_SymlinkEscape_ReturnsForbidden(t *testing.T) {
setupTestToken(t, "")
setupTestNoAuthFallback(t, true)
h, workspace := makeTestFilesHandler(t)
outsideDir := t.TempDir()
target := filepath.Join(outsideDir, "secret.txt")
if err := os.WriteFile(target, []byte("secret"), 0644); err != nil {
t.Fatal(err)
}
linkPath := filepath.Join(workspace, "link.txt")
if err := os.Symlink(target, linkPath); err != nil {
t.Skipf("symlink unavailable: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/v1/files/sign", strings.NewReader(`{"path":`+strconv.Quote(linkPath)+`}`))
w := httptest.NewRecorder()
h.handleSign(w, req)
if w.Code == http.StatusOK {
t.Fatal("sign endpoint should reject symlinks escaping allowed roots")
}
}
// ---- handleServe: empty path ----
func TestFilesHandleServe_EmptyPath_Returns400(t *testing.T) {
internal/http/gateway_upgrade.go
+262
View File
@@ -0,0 +1,262 @@
package http
import (
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"github.com/nextlevelbuilder/goclaw/internal/permissions"
)
const (
defaultGatewayUpgradeScript = "/usr/local/bin/goclaw-upgrade-release"
defaultGatewayUpgradeStatus = "/var/lib/goclaw/update-jobs/current.json"
gatewayUpgradeTokenHeader = "X-GoClaw-Upgrade-Token"
)
var gatewayUpgradeTagRE = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+(-(beta|rc)\.[0-9]+)?$`)
type gatewayUpgradeRunner interface {
Start(tag string) error
}
type gatewayUpgradeCommandRunner struct {
scriptPath string
}
func (r gatewayUpgradeCommandRunner) Start(tag string) error {
cmd := exec.Command("sudo", "-n", r.scriptPath, tag)
if err := cmd.Start(); err != nil {
return err
}
go func() {
if err := cmd.Wait(); err != nil {
slog.Warn("gateway upgrade command exited with error", "error", err)
}
}()
return nil
}
// GatewayUpgradeHandler triggers the host-local GoClaw release upgrade script.
// It never accepts arbitrary commands or URLs.
type GatewayUpgradeHandler struct {
ScriptPath string
StatusPath string
TriggerToken string
Runner gatewayUpgradeRunner
mu sync.Mutex
}
func NewGatewayUpgradeHandlerFromEnv() *GatewayUpgradeHandler {
scriptPath := strings.TrimSpace(os.Getenv("GOCLAW_UPGRADE_SCRIPT"))
if scriptPath == "" {
scriptPath = defaultGatewayUpgradeScript
}
statusPath := strings.TrimSpace(os.Getenv("GOCLAW_UPGRADE_STATUS_PATH"))
if statusPath == "" {
statusPath = defaultGatewayUpgradeStatus
}
h := &GatewayUpgradeHandler{
ScriptPath: scriptPath,
StatusPath: statusPath,
TriggerToken: os.Getenv("GOCLAW_UPGRADE_TRIGGER_TOKEN"),
}
h.Runner = gatewayUpgradeCommandRunner{scriptPath: h.ScriptPath}
return h
}
func (h *GatewayUpgradeHandler) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("GET /v1/system/gateway/upgrade/status", requireAuth(permissions.RoleAdmin, h.handleStatus))
mux.HandleFunc("POST /v1/system/gateway/upgrade", requireAuth(permissions.RoleAdmin, h.handleStart))
}
func (h *GatewayUpgradeHandler) handleStatus(w http.ResponseWriter, r *http.Request) {
if !requireMasterScope(w, r) {
return
}
if !h.requireTriggerToken(w, r) {
return
}
status, err := h.readStatus()
if err != nil {
slog.Error("gateway upgrade status read failed", "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "failed to read gateway upgrade status"})
return
}
writeJSON(w, http.StatusOK, status)
}
func (h *GatewayUpgradeHandler) handleStart(w http.ResponseWriter, r *http.Request) {
if !requireMasterScope(w, r) {
return
}
if !enforcePackagesWriteLimit(w, r, "/v1/system/gateway/upgrade") {
return
}
if !h.requireTriggerToken(w, r) {
return
}
r.Body = http.MaxBytesReader(w, r.Body, 1024)
var req struct {
Tag string `json:"tag"`
}
if !bindJSON(w, r, extractLocale(r), &req) {
return
}
tag := strings.TrimSpace(req.Tag)
if !validGatewayUpgradeTag(tag) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "tag must be latest or vMAJOR.MINOR.PATCH[-beta.N|-rc.N]"})
return
}
h.mu.Lock()
defer h.mu.Unlock()
status, err := h.readStatus()
if err != nil {
slog.Error("gateway upgrade status read failed", "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "failed to read gateway upgrade status"})
return
}
if status["state"] == "running" {
writeJSON(w, http.StatusConflict, map[string]string{"error": "gateway upgrade already running"})
return
}
runner := h.Runner
if runner == nil {
runner = gatewayUpgradeCommandRunner{scriptPath: h.ScriptPath}
}
if err := h.writeRunningStatus(tag); err != nil {
slog.Error("gateway upgrade status write failed", "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "failed to write gateway upgrade status"})
return
}
if err := runner.Start(tag); err != nil {
slog.Error("gateway upgrade start failed", "error", err)
_ = h.writeFailedStatus(tag, "failed to start gateway upgrade")
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "failed to start gateway upgrade"})
return
}
writeJSON(w, http.StatusAccepted, map[string]any{
"ok": true,
"accepted": true,
"tag": tag,
})
}
func (h *GatewayUpgradeHandler) requireTriggerToken(w http.ResponseWriter, r *http.Request) bool {
if h.TriggerToken == "" {
slog.Warn("security.gateway_upgrade_token_unconfigured", "path", r.URL.Path)
writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "gateway upgrade trigger token is not configured"})
return false
}
provided := r.Header.Get(gatewayUpgradeTokenHeader)
if subtle.ConstantTimeCompare([]byte(provided), []byte(h.TriggerToken)) == 1 {
return true
}
slog.Warn("security.gateway_upgrade_token_denied", "path", r.URL.Path)
writeJSON(w, http.StatusForbidden, map[string]string{"error": "upgrade trigger token required"})
return false
}
func (h *GatewayUpgradeHandler) readStatus() (map[string]any, error) {
path := h.statusPath()
data, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
return map[string]any{"state": "idle"}, nil
}
if err != nil {
return nil, fmt.Errorf("read upgrade status: %w", err)
}
if len(data) > 64*1024 {
return nil, fmt.Errorf("upgrade status too large")
}
var status map[string]any
if err := json.Unmarshal(data, &status); err != nil {
return nil, fmt.Errorf("decode upgrade status: %w", err)
}
if status == nil {
return map[string]any{"state": "idle"}, nil
}
return status, nil
}
func (h *GatewayUpgradeHandler) writeRunningStatus(tag string) error {
return h.writeStatus(map[string]any{
"jobId": time.Now().UTC().Format("20060102T150405Z") + "-" + tag,
"state": "running",
"requestedTag": tag,
"resolvedTag": "",
"startedAt": time.Now().UTC().Format(time.RFC3339),
"finishedAt": nil,
"error": nil,
})
}
func (h *GatewayUpgradeHandler) writeFailedStatus(tag, reason string) error {
return h.writeStatus(map[string]any{
"jobId": time.Now().UTC().Format("20060102T150405Z") + "-" + tag,
"state": "failed",
"requestedTag": tag,
"resolvedTag": "",
"startedAt": time.Now().UTC().Format(time.RFC3339),
"finishedAt": time.Now().UTC().Format(time.RFC3339),
"error": reason,
})
}
func (h *GatewayUpgradeHandler) writeStatus(status map[string]any) error {
path := h.statusPath()
if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil {
return fmt.Errorf("create upgrade status dir: %w", err)
}
tmp, err := os.CreateTemp(filepath.Dir(path), ".current-*.json")
if err != nil {
return fmt.Errorf("create upgrade status tmp: %w", err)
}
tmpName := tmp.Name()
encErr := json.NewEncoder(tmp).Encode(status)
closeErr := tmp.Close()
if encErr != nil {
_ = os.Remove(tmpName)
return fmt.Errorf("encode upgrade status: %w", encErr)
}
if closeErr != nil {
_ = os.Remove(tmpName)
return fmt.Errorf("close upgrade status tmp: %w", closeErr)
}
if err := os.Chmod(tmpName, 0o640); err != nil {
_ = os.Remove(tmpName)
return fmt.Errorf("chmod upgrade status tmp: %w", err)
}
if err := os.Rename(tmpName, path); err != nil {
_ = os.Remove(tmpName)
return fmt.Errorf("replace upgrade status: %w", err)
}
return nil
}
func (h *GatewayUpgradeHandler) statusPath() string {
if h.StatusPath == "" {
return defaultGatewayUpgradeStatus
}
return filepath.Clean(h.StatusPath)
}
func validGatewayUpgradeTag(tag string) bool {
return tag == "latest" || gatewayUpgradeTagRE.MatchString(tag)
}
internal/http/gateway_upgrade_test.go
+280
View File
@@ -0,0 +1,280 @@
package http
import (
"bytes"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
)
type fakeGatewayUpgradeRunner struct {
tags []string
err error
}
func (r *fakeGatewayUpgradeRunner) Start(tag string) error {
r.tags = append(r.tags, tag)
return r.err
}
func TestValidGatewayUpgradeTag(t *testing.T) {
tests := []struct {
tag string
want bool
}{
{"latest", true},
{"v3.12.0", true},
{"v3.12.0-beta.1", true},
{"v3.12.0-rc.2", true},
{"", false},
{"3.12.0", false},
{"v3.12", false},
{"v3.12.0-beta", false},
{"https://example.com/goclaw.tar.gz", false},
{"v3.12.0;reboot", false},
{"../v3.12.0", false},
{"v3.12.0 linux", false},
}
for _, tt := range tests {
if got := validGatewayUpgradeTag(tt.tag); got != tt.want {
t.Fatalf("validGatewayUpgradeTag(%q) = %v, want %v", tt.tag, got, tt.want)
}
}
}
func TestGatewayUpgradeStatusMissingReturnsIdle(t *testing.T) {
h := &GatewayUpgradeHandler{StatusPath: filepath.Join(t.TempDir(), "missing.json"), TriggerToken: "secret-token"}
req := httptest.NewRequest(http.MethodGet, "/v1/system/gateway/upgrade/status", nil)
req.Header.Set(gatewayUpgradeTokenHeader, "secret-token")
req = req.WithContext(ownerCtx(req.Context(), "gateway-status-owner"))
w := httptest.NewRecorder()
h.handleStatus(w, req)
if w.Code != http.StatusOK {
t.Fatalf("want 200, got %d: %s", w.Code, w.Body.String())
}
var body map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if body["state"] != "idle" {
t.Fatalf("state = %v, want idle", body["state"])
}
}
func TestGatewayUpgradeStartAcceptsValidTag(t *testing.T) {
runner := &fakeGatewayUpgradeRunner{}
h := &GatewayUpgradeHandler{
StatusPath: filepath.Join(t.TempDir(), "status.json"),
TriggerToken: "secret-token",
Runner: runner,
}
req := httptest.NewRequest(http.MethodPost, "/v1/system/gateway/upgrade", bytes.NewBufferString(`{"tag":"v3.12.0"}`))
req.Header.Set(gatewayUpgradeTokenHeader, "secret-token")
req = req.WithContext(ownerCtx(req.Context(), "gateway-start-owner"))
w := httptest.NewRecorder()
h.handleStart(w, req)
if w.Code != http.StatusAccepted {
t.Fatalf("want 202, got %d: %s", w.Code, w.Body.String())
}
if len(runner.tags) != 1 || runner.tags[0] != "v3.12.0" {
t.Fatalf("runner tags = %#v, want [v3.12.0]", runner.tags)
}
}
func TestGatewayUpgradeStartRejectsInvalidTag(t *testing.T) {
runner := &fakeGatewayUpgradeRunner{}
h := &GatewayUpgradeHandler{
StatusPath: filepath.Join(t.TempDir(), "status.json"),
TriggerToken: "secret-token",
Runner: runner,
}
req := httptest.NewRequest(http.MethodPost, "/v1/system/gateway/upgrade", bytes.NewBufferString(`{"tag":"https://example.com/x"}`))
req.Header.Set(gatewayUpgradeTokenHeader, "secret-token")
req = req.WithContext(ownerCtx(req.Context(), "gateway-invalid-owner"))
w := httptest.NewRecorder()
h.handleStart(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("want 400, got %d: %s", w.Code, w.Body.String())
}
if len(runner.tags) != 0 {
t.Fatalf("runner should not be called, got %#v", runner.tags)
}
}
func TestGatewayUpgradeStartRejectsRunningJob(t *testing.T) {
dir := t.TempDir()
statusPath := filepath.Join(dir, "status.json")
if err := os.WriteFile(statusPath, []byte(`{"state":"running"}`), 0o600); err != nil {
t.Fatalf("write status: %v", err)
}
runner := &fakeGatewayUpgradeRunner{}
h := &GatewayUpgradeHandler{StatusPath: statusPath, TriggerToken: "secret-token", Runner: runner}
req := httptest.NewRequest(http.MethodPost, "/v1/system/gateway/upgrade", bytes.NewBufferString(`{"tag":"latest"}`))
req.Header.Set(gatewayUpgradeTokenHeader, "secret-token")
req = req.WithContext(ownerCtx(req.Context(), "gateway-running-owner"))
w := httptest.NewRecorder()
h.handleStart(w, req)
if w.Code != http.StatusConflict {
t.Fatalf("want 409, got %d: %s", w.Code, w.Body.String())
}
if len(runner.tags) != 0 {
t.Fatalf("runner should not be called, got %#v", runner.tags)
}
}
func TestGatewayUpgradeTriggerTokenGuard(t *testing.T) {
runner := &fakeGatewayUpgradeRunner{}
h := &GatewayUpgradeHandler{
StatusPath: filepath.Join(t.TempDir(), "status.json"),
TriggerToken: "secret-token",
Runner: runner,
}
req := httptest.NewRequest(http.MethodPost, "/v1/system/gateway/upgrade", bytes.NewBufferString(`{"tag":"latest"}`))
req = req.WithContext(ownerCtx(req.Context(), "gateway-token-owner"))
w := httptest.NewRecorder()
h.handleStart(w, req)
if w.Code != http.StatusForbidden {
t.Fatalf("want 403, got %d: %s", w.Code, w.Body.String())
}
if len(runner.tags) != 0 {
t.Fatalf("runner should not be called, got %#v", runner.tags)
}
}
func TestGatewayUpgradeRejectsNonMasterScope(t *testing.T) {
runner := &fakeGatewayUpgradeRunner{}
h := &GatewayUpgradeHandler{
StatusPath: filepath.Join(t.TempDir(), "status.json"),
TriggerToken: "secret-token",
Runner: runner,
}
req := httptest.NewRequest(http.MethodPost, "/v1/system/gateway/upgrade", bytes.NewBufferString(`{"tag":"latest"}`))
req.Header.Set(gatewayUpgradeTokenHeader, "secret-token")
req = req.WithContext(tenantAdminCtx(req.Context(), "gateway-tenant-admin"))
w := httptest.NewRecorder()
h.handleStart(w, req)
if w.Code != http.StatusForbidden {
t.Fatalf("want 403, got %d: %s", w.Code, w.Body.String())
}
if len(runner.tags) != 0 {
t.Fatalf("runner should not be called, got %#v", runner.tags)
}
}
func TestGatewayUpgradeStartRunnerError(t *testing.T) {
h := &GatewayUpgradeHandler{
StatusPath: filepath.Join(t.TempDir(), "status.json"),
TriggerToken: "secret-token",
Runner: &fakeGatewayUpgradeRunner{err: errors.New("boom")},
}
req := httptest.NewRequest(http.MethodPost, "/v1/system/gateway/upgrade", bytes.NewBufferString(`{"tag":"latest"}`))
req.Header.Set(gatewayUpgradeTokenHeader, "secret-token")
req = req.WithContext(ownerCtx(req.Context(), "gateway-runner-error-owner"))
w := httptest.NewRecorder()
h.handleStart(w, req)
if w.Code != http.StatusInternalServerError {
t.Fatalf("want 500, got %d: %s", w.Code, w.Body.String())
}
}
func TestGatewayUpgradeFailsClosedWithoutConfiguredTriggerToken(t *testing.T) {
runner := &fakeGatewayUpgradeRunner{}
h := &GatewayUpgradeHandler{
StatusPath: filepath.Join(t.TempDir(), "status.json"),
Runner: runner,
}
req := httptest.NewRequest(http.MethodPost, "/v1/system/gateway/upgrade", bytes.NewBufferString(`{"tag":"latest"}`))
req = req.WithContext(ownerCtx(req.Context(), "gateway-no-token-owner"))
w := httptest.NewRecorder()
h.handleStart(w, req)
if w.Code != http.StatusServiceUnavailable {
t.Fatalf("want 503, got %d: %s", w.Code, w.Body.String())
}
if len(runner.tags) != 0 {
t.Fatalf("runner should not be called, got %#v", runner.tags)
}
}
func TestGatewayUpgradeRegisterRoutes(t *testing.T) {
runner := &fakeGatewayUpgradeRunner{}
h := &GatewayUpgradeHandler{
StatusPath: filepath.Join(t.TempDir(), "status.json"),
TriggerToken: "secret-token",
Runner: runner,
}
mux := http.NewServeMux()
h.RegisterRoutes(mux)
req := httptest.NewRequest(http.MethodPost, "/v1/system/gateway/upgrade", bytes.NewBufferString(`{"tag":"latest"}`))
req.Header.Set(gatewayUpgradeTokenHeader, "secret-token")
w := httptest.NewRecorder()
mux.ServeHTTP(w, req)
if w.Code != http.StatusAccepted {
t.Fatalf("want 202, got %d: %s", w.Code, w.Body.String())
}
if len(runner.tags) != 1 || runner.tags[0] != "latest" {
t.Fatalf("runner tags = %#v, want [latest]", runner.tags)
}
}
func TestNewGatewayUpgradeHandlerFromEnvDefaults(t *testing.T) {
t.Setenv("GOCLAW_UPGRADE_SCRIPT", "")
t.Setenv("GOCLAW_UPGRADE_STATUS_PATH", "")
t.Setenv("GOCLAW_UPGRADE_TRIGGER_TOKEN", "secret-token")
h := NewGatewayUpgradeHandlerFromEnv()
if h.ScriptPath != defaultGatewayUpgradeScript {
t.Fatalf("ScriptPath = %q, want %q", h.ScriptPath, defaultGatewayUpgradeScript)
}
if h.StatusPath != defaultGatewayUpgradeStatus {
t.Fatalf("StatusPath = %q, want %q", h.StatusPath, defaultGatewayUpgradeStatus)
}
if h.TriggerToken != "secret-token" {
t.Fatalf("TriggerToken was not loaded from env")
}
}
func TestGatewayUpgradeStatusRejectsInvalidJSON(t *testing.T) {
statusPath := filepath.Join(t.TempDir(), "status.json")
if err := os.WriteFile(statusPath, []byte(`{bad json`), 0o600); err != nil {
t.Fatalf("write status: %v", err)
}
h := &GatewayUpgradeHandler{StatusPath: statusPath, TriggerToken: "secret-token"}
req := httptest.NewRequest(http.MethodGet, "/v1/system/gateway/upgrade/status", nil)
req.Header.Set(gatewayUpgradeTokenHeader, "secret-token")
req = req.WithContext(ownerCtx(req.Context(), "gateway-invalid-status-owner"))
w := httptest.NewRecorder()
h.handleStatus(w, req)
if w.Code != http.StatusInternalServerError {
t.Fatalf("want 500, got %d: %s", w.Code, w.Body.String())
}
if strings.Contains(w.Body.String(), statusPath) || strings.Contains(w.Body.String(), "decode upgrade status") {
t.Fatalf("response leaked internal status details: %s", w.Body.String())
}
}
internal/http/openapi_spec.json
+1 -1
View File
@@ -2,7 +2,7 @@
"openapi": "3.0.3",
"info": {
"title": "GoClaw Gateway API",
"description": "PostgreSQL multi-tenant AI agent gateway with WebSocket RPC + HTTP API.\n\n## Authentication\n\nAll endpoints require a Bearer token in the `Authorization` header:\n\n```\nAuthorization: Bearer <gateway-token-or-api-key>\n```\n\nYou can use either the **gateway token** (grants admin access) or an **API key** created via the API Keys endpoints (grants scoped access).\n\nIf no token is configured on the server, authentication is disabled (backward compatibility).\n\n## Common Headers\n\n| Header | Description |\n|--------|-------------|\n| `X-GoClaw-User-Id` | External user ID for multi-tenant context |\n| `X-GoClaw-Agent-Id` | Target agent ID (alternative to model prefix) |\n| `X-GoClaw-Tenant-Id` | Tenant scope — UUID or slug (gateway token / cross-tenant API keys) |\n| `Accept-Language` | Locale for error messages (`en`, `vi`, `zh`) |\n\n## WebSocket Protocol\n\nConnect via `POST /ws` (upgrade). Protocol v3 uses frame types: `req`, `res`, `event`.\nFirst request must be `connect` with `{\"token\": \"...\", \"user_id\": \"...\", \"locale\": \"en\"}`.",
"description": "PostgreSQL multi-tenant AI agent gateway with WebSocket RPC + HTTP API.\n\n## Authentication\n\nAll endpoints require a Bearer token in the `Authorization` header:\n\n```\nAuthorization: Bearer <gateway-token-or-api-key>\n```\n\nYou can use either the **gateway token** (grants admin access) or an **API key** created via the API Keys endpoints (grants scoped access).\n\nIf no gateway token is configured, empty-token admin access is limited to loopback local development or explicit `GOCLAW_ALLOW_INSECURE_NO_AUTH=1`; externally reachable deployments require `GOCLAW_GATEWAY_TOKEN`.\n\n## Common Headers\n\n| Header | Description |\n|--------|-------------|\n| `X-GoClaw-User-Id` | External user ID for multi-tenant context |\n| `X-GoClaw-Agent-Id` | Target agent ID (alternative to model prefix) |\n| `X-GoClaw-Tenant-Id` | Tenant scope — UUID or slug (gateway token / cross-tenant API keys) |\n| `Accept-Language` | Locale for error messages (`en`, `vi`, `zh`) |\n\n## WebSocket Protocol\n\nConnect via `POST /ws` (upgrade). Protocol v3 uses frame types: `req`, `res`, `event`.\nFirst request must be `connect` with `{\"token\": \"...\", \"user_id\": \"...\", \"locale\": \"en\"}`.",
"version": "0.2.0",
"contact": {
"name": "GoClaw",
internal/http/packages.go
+13 -3
View File
@@ -10,6 +10,7 @@ import (
"strconv"
"strings"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/internal/skills"
"github.com/nextlevelbuilder/goclaw/internal/tools"
@@ -31,11 +32,15 @@ var validGitHubBareName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
var validRepoPath = regexp.MustCompile(`^([A-Za-z0-9](?:[A-Za-z0-9-]{0,37})?[A-Za-z0-9]|[A-Za-z0-9])/[A-Za-z0-9][A-Za-z0-9._-]*$`)
// PackagesHandler handles runtime package management HTTP endpoints.
type PackagesHandler struct{}
type PackagesHandler struct {
Registry *skills.UpdateRegistry
Publisher bus.EventPublisher
}
// NewPackagesHandler creates a handler for package management endpoints.
func NewPackagesHandler() *PackagesHandler {
return &PackagesHandler{}
// Pass nil registry/publisher for read-only mode (no update endpoints).
func NewPackagesHandler(registry *skills.UpdateRegistry, publisher bus.EventPublisher) *PackagesHandler {
return &PackagesHandler{Registry: registry, Publisher: publisher}
}
// RegisterRoutes registers all package management routes on the given mux.
@@ -46,6 +51,11 @@ func (h *PackagesHandler) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("GET /v1/packages/runtimes", h.readAuth(h.handleRuntimes))
mux.HandleFunc("GET /v1/packages/github-releases", h.readAuth(h.handleGitHubReleases))
mux.HandleFunc("GET /v1/shell-deny-groups", h.readAuth(h.handleDenyGroups))
// Update flow (Phase 4+5) — operator+ read, admin+master-scope writes.
mux.HandleFunc("GET /v1/packages/updates", h.readAuth(h.handleListUpdates))
mux.HandleFunc("POST /v1/packages/updates/refresh", h.adminAuth(h.handleRefreshUpdates))
mux.HandleFunc("POST /v1/packages/update", h.adminAuth(h.handleUpdatePackage))
mux.HandleFunc("POST /v1/packages/updates/apply-all", h.adminAuth(h.handleApplyAllUpdates))
}
// readAuth allows viewer+ for read operations.
internal/http/packages_test.go
+2 -2
View File
@@ -216,7 +216,7 @@ func TestParseAndValidatePackage_BodySizeLimit(t *testing.T) {
// TestNewPackagesHandler creates a handler.
func TestNewPackagesHandler(t *testing.T) {
h := NewPackagesHandler()
h := NewPackagesHandler(nil, nil)
if h == nil {
t.Fatal("NewPackagesHandler() returned nil")
}
@@ -224,7 +224,7 @@ func TestNewPackagesHandler(t *testing.T) {
// TestPackagesHandler_RegisterRoutes ensures routes are registered without panic.
func TestPackagesHandler_RegisterRoutes(t *testing.T) {
h := NewPackagesHandler()
h := NewPackagesHandler(nil, nil)
mux := http.NewServeMux()
// Should not panic
Loaded 100 of 455 files, more files were not shown because too many files have changed in this diff. Show more