mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-13 16:14:57 +00:00
* feat(packages): add update flow for GitHub binaries (#900) Closes #900. Proactive update-check + atomic swap for GitHub-installed binaries on the Runtime & Packages page. Interfaces prepared for pip/npm/apk extension in Phase 2. - UpdateCache + UpdateRegistry + PackageLocker (ctx-aware keyed mutex) - GitHubUpdateChecker: ETag-aware, distinct /latest vs /list ETag keys, semver-correct ordering via golang.org/x/mod/semver, non-semver fallback that refuses to downgrade, pre-release + stable candidate fusion for the v1.0.0-rc.1 -> v1.0.0 transition - GitHubUpdateExecutor: two-phase .bak swap with hadBackup-aware rollback, manifest save retry (3x, 100ms/500ms/1s backoff), nil-safe meta access, explicit ScratchDir, 0755 set pre-rename - HTTP: GET /v1/packages/updates (SWR), POST /v1/packages/updates/refresh, POST /v1/packages/update, POST /v1/packages/updates/apply-all (always 200, failed[] is error source). Master-scope gated. - WS events package.update.{checked,started,succeeded,failed} forwarded to owner clients via event_filter.go - Frontend: useUpdates hook + 3 components (summary bar, update-all modal, row button), master-scope-gated disabled state - i18n: 8 backend keys + 17 frontend keys x en/vi/zh - Config: packages.github_token (reserved), updates_check_ttl, scratch_dir - 45+ new tests, race-clean, BenchmarkCheckAll10Packages ~1.1ms/op warm * docs(packages): document update flow + Phase 1 completion - packages-github.md: "Updating Installed Packages" section with UI + API contract, troubleshooting runbook (corrupt cache, rate-limit, scratch dir, mid-swap recovery) - 17-changelog.md + CHANGELOG.md: Phase 1 entry - 14-skills-runtime.md: cross-ref to update flow - journal entry capturing CRIT fixes (double-write, lock-key mismatch, rollback false-alarm) + design wins (keyed locks, red-team pre-flight) * feat(workstation): remote workstation runtime — SSH exec + security + audit Adds generic Remote Workstation Runtime enabling agents to execute commands on user-owned SSH workstations. Includes registry (DB + API + UI), SSH backend with connection pool and circuit breaker, workstation.exec + claude_remote tools, NFKC + binary-name allowlist security, and audit logging. Standard edition only. Closes #941. * fix(workstation): address 3 critical + 5 important code review findings - C1: Add json:"-" to Metadata/DefaultEnv fields; use SanitizedView() in all API responses to prevent SSH private key leakage - C2: Wire CheckEnv into PermCheckFn; LD_PRELOAD/PATH injection now blocked - C3: SSH Setenv fallback — prepend `export K=V;` when server rejects Setenv - I1: BackendCache sync.RWMutex → sync.Mutex (fix data race on lastUsed) - I2: Validate metadata shape in handleUpdate before store write - I3: Include command in exec-done event; activity sink uses actual cmd hash - I4: Wrap pool release in sync.Once (idempotent double-call safety) - I5: Verify workstation tenant ownership before adding permissions * fix(packages): bypass HTTPS+IP validation in update executor tests Test httptest servers bind to http://127.0.0.1 which fails both the HTTPS scheme check and literal-IP SSRF guard. Add testSkipDownloadValidation flag (same pattern as existing withTestDownloadHosts) to skip full URL validation in test context. * fix(workstation): address Claude review findings — tenant isolation + pool leak + dead code - Activity list: add workstation ownership check before listing (prevents cross-tenant activity enumeration via known UUID) - SSH pool: clean up p.sem + p.circuits maps in CloseWorkstation, prune, and Close to prevent unbounded map growth - RPC handlers: return ErrInvalidRequest on JSON unmarshal failure instead of silently using zero-value params - Remove unused containsControlChars function in normalize.go - HTTP tests: add 10s context timeout to prevent CI package timeout * fix(workstation): DefaultEnv JSON parse, backend cache leak, perm ownership check - DefaultEnv: replace KEY=VALUE text parse with json.Unmarshal (stored as JSON by HTTP handler, was silently ignored) - BackendCache: close losing backend on concurrent cache miss to prevent pruneLoop goroutine leak - Backend interface: add Close() error method; SSHBackend delegates to pool.Close() - handlePermList: add wsStore.GetByID ownership check (prevents cross-tenant UUID enumeration returning empty array vs 404) - scanRows: log scan errors instead of silently skipping * fix(workstation): wire activity sink shutdown + remove misleading comment - WireActivitySink: capture cleanup func, register in gateway shutdown (was discarded → retention goroutine leaked + buffered rows lost) - Add Stop() to WorkstationActivityStore interface (PG+SQLite already had it) - wireWorkstationTools returns cleanup func; gateway.go defers it - Remove misleading "re-validate env" comment in allowlist.go Check() * ci: bump unit test timeout from 90s to 120s hooks/handlers package (goja script tests) consumes ~85s on cold CI runners, leaving insufficient headroom for HTTP retry tests with 1s backoff. 120s provides adequate breathing room without masking real deadlocks. * fix: compile errors in integration tests + allowlist docstring - packages_update_test: add missing lockKey arg to registry.Apply - mcp_grant_revoke_test: remove unused fakeMCPClient struct - allowlist.go: fix Check() docstring to match actual 3-step pipeline * fix(test): relax mcp grant revoke assertion for pre-Phase02 state Execute-time grant checking not yet wired — test correctly gets an error but the message is "no active client" (nil clientPtr) rather than "grant revoked". Accept any error as valid regression guard. * chore: trigger CI on digitopvn/goclaw fork * ci: retrigger workflows * fix(permissions): classify workstation methods in RBAC policy
208 lines
5.4 KiB
Go
208 lines
5.4 KiB
Go
package pg
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"log/slog"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/nextlevelbuilder/goclaw/internal/store"
|
|
)
|
|
|
|
const (
|
|
activityBufferSize = 1000
|
|
activityBatchMax = 100
|
|
activityFlushPeriod = 500 * time.Millisecond
|
|
)
|
|
|
|
// PGWorkstationActivityStore implements store.WorkstationActivityStore backed by Postgres.
|
|
// Inserts are buffered (channel size 1000) and flushed in batches every 500ms or 100 rows,
|
|
// keeping exec hot-path latency below 1ms.
|
|
type PGWorkstationActivityStore struct {
|
|
db *sql.DB
|
|
buf chan *store.WorkstationActivity
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
// NewPGWorkstationActivityStore creates the store and starts the background flush goroutine.
|
|
func NewPGWorkstationActivityStore(db *sql.DB) *PGWorkstationActivityStore {
|
|
s := &PGWorkstationActivityStore{
|
|
db: db,
|
|
buf: make(chan *store.WorkstationActivity, activityBufferSize),
|
|
}
|
|
s.wg.Add(1)
|
|
go s.flusher()
|
|
return s
|
|
}
|
|
|
|
// Insert enqueues the row for async batch insert. Drops and warns if buffer is full.
|
|
func (s *PGWorkstationActivityStore) Insert(_ context.Context, row *store.WorkstationActivity) error {
|
|
select {
|
|
case s.buf <- row:
|
|
default:
|
|
slog.Warn("workstation.activity.buffer_full", "action", row.Action)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// List returns up to limit rows for the workstation, newest first.
|
|
// Cursor-based pagination: pass last seen ID to continue from that point.
|
|
func (s *PGWorkstationActivityStore) List(ctx context.Context, workstationID uuid.UUID, limit int, cursor *uuid.UUID) ([]store.WorkstationActivity, *uuid.UUID, error) {
|
|
if limit <= 0 || limit > 200 {
|
|
limit = 50
|
|
}
|
|
|
|
var rows *sql.Rows
|
|
var err error
|
|
if cursor == nil {
|
|
rows, err = s.db.QueryContext(ctx,
|
|
`SELECT id, tenant_id, workstation_id, agent_id, action, cmd_hash, cmd_preview,
|
|
exit_code, duration_ms, deny_reason, created_at
|
|
FROM workstation_activity
|
|
WHERE workstation_id = $1
|
|
ORDER BY created_at DESC
|
|
LIMIT $2`,
|
|
workstationID, limit+1,
|
|
)
|
|
} else {
|
|
// Cursor: created_at of the cursor row acts as the page boundary.
|
|
rows, err = s.db.QueryContext(ctx,
|
|
`SELECT id, tenant_id, workstation_id, agent_id, action, cmd_hash, cmd_preview,
|
|
exit_code, duration_ms, deny_reason, created_at
|
|
FROM workstation_activity
|
|
WHERE workstation_id = $1
|
|
AND created_at < (SELECT created_at FROM workstation_activity WHERE id = $2)
|
|
ORDER BY created_at DESC
|
|
LIMIT $3`,
|
|
workstationID, *cursor, limit+1,
|
|
)
|
|
}
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var result []store.WorkstationActivity
|
|
for rows.Next() {
|
|
var a store.WorkstationActivity
|
|
if err := rows.Scan(
|
|
&a.ID, &a.TenantID, &a.WorkstationID, &a.AgentID, &a.Action,
|
|
&a.CmdHash, &a.CmdPreview, &a.ExitCode, &a.DurationMS, &a.DenyReason, &a.CreatedAt,
|
|
); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
result = append(result, a)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
var nextCursor *uuid.UUID
|
|
if len(result) > limit {
|
|
last := result[limit-1].ID
|
|
nextCursor = &last
|
|
result = result[:limit]
|
|
}
|
|
return result, nextCursor, nil
|
|
}
|
|
|
|
// Prune deletes rows created before the given time in batches to avoid long locks.
|
|
// Returns total rows deleted.
|
|
func (s *PGWorkstationActivityStore) Prune(ctx context.Context, before time.Time) (int64, error) {
|
|
var total int64
|
|
for {
|
|
res, err := s.db.ExecContext(ctx,
|
|
`DELETE FROM workstation_activity
|
|
WHERE id IN (
|
|
SELECT id FROM workstation_activity WHERE created_at < $1 LIMIT 1000
|
|
)`,
|
|
before,
|
|
)
|
|
if err != nil {
|
|
return total, err
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
total += n
|
|
if n < 1000 {
|
|
break
|
|
}
|
|
// Brief sleep between batches to reduce lock pressure.
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
return total, nil
|
|
}
|
|
|
|
// flusher reads from buf and batch-inserts into the DB every 500ms or 100 rows.
|
|
func (s *PGWorkstationActivityStore) flusher() {
|
|
defer s.wg.Done()
|
|
ticker := time.NewTicker(activityFlushPeriod)
|
|
defer ticker.Stop()
|
|
|
|
var batch []*store.WorkstationActivity
|
|
flush := func() {
|
|
if len(batch) == 0 {
|
|
return
|
|
}
|
|
if err := s.batchInsert(context.Background(), batch); err != nil {
|
|
slog.Warn("workstation.activity.flush_error", "error", err, "count", len(batch))
|
|
}
|
|
batch = batch[:0]
|
|
}
|
|
|
|
for {
|
|
select {
|
|
case row, ok := <-s.buf:
|
|
if !ok {
|
|
flush()
|
|
return
|
|
}
|
|
batch = append(batch, row)
|
|
if len(batch) >= activityBatchMax {
|
|
flush()
|
|
}
|
|
case <-ticker.C:
|
|
flush()
|
|
}
|
|
}
|
|
}
|
|
|
|
// batchInsert inserts rows using individual statements (no unnest for portability).
|
|
func (s *PGWorkstationActivityStore) batchInsert(ctx context.Context, rows []*store.WorkstationActivity) error {
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
stmt, err := tx.PrepareContext(ctx,
|
|
`INSERT INTO workstation_activity
|
|
(id, tenant_id, workstation_id, agent_id, action, cmd_hash, cmd_preview,
|
|
exit_code, duration_ms, deny_reason, created_at)
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
|
|
ON CONFLICT (id) DO NOTHING`,
|
|
)
|
|
if err != nil {
|
|
_ = tx.Rollback()
|
|
return err
|
|
}
|
|
defer stmt.Close()
|
|
|
|
for _, r := range rows {
|
|
if _, err := stmt.ExecContext(ctx,
|
|
r.ID, r.TenantID, r.WorkstationID, r.AgentID, r.Action,
|
|
r.CmdHash, r.CmdPreview, r.ExitCode, r.DurationMS, r.DenyReason, r.CreatedAt,
|
|
); err != nil {
|
|
_ = tx.Rollback()
|
|
return err
|
|
}
|
|
}
|
|
return tx.Commit()
|
|
}
|
|
|
|
// Stop drains the buffer and shuts down the flush goroutine.
|
|
func (s *PGWorkstationActivityStore) Stop() {
|
|
close(s.buf)
|
|
s.wg.Wait()
|
|
}
|