Merge remote-tracking branch 'origin/dev' into codex/issue-48-google-workspace-cli

# Conflicts:
#	docs/project-changelog.md
This commit is contained in:
Goon committed 2026-05-24 18:57:47 +07:00
commit 3a749ff91c
189 files changed
+8860 -848

No files matched your search

+9 -1
View File
@@ -7,12 +7,14 @@ import (
kg "github.com/nextlevelbuilder/goclaw/internal/knowledgegraph"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// KnowledgeGraphHandler handles KG entity/relation management endpoints.
type KnowledgeGraphHandler struct {
store store.KnowledgeGraphStore
providerReg *providers.Registry
usageCaps *usagecaps.Service
}
// NewKnowledgeGraphHandler creates a handler for KG management endpoints.
@@ -20,6 +22,10 @@ func NewKnowledgeGraphHandler(s store.KnowledgeGraphStore, providerReg *provider
return &KnowledgeGraphHandler{store: s, providerReg: providerReg}
}
func (h *KnowledgeGraphHandler) SetUsageCapService(s *usagecaps.Service) {
h.usageCaps = s
}
// NewExtractor creates an Extractor from the given provider name and model.
func (h *KnowledgeGraphHandler) NewExtractor(ctx context.Context, providerName, model string, minConfidence float64) *kg.Extractor {
if h.providerReg == nil || providerName == "" || model == "" {
@@ -29,7 +35,9 @@ func (h *KnowledgeGraphHandler) NewExtractor(ctx context.Context, providerName,
if err != nil {
return nil
}
return kg.NewExtractor(p, model, minConfidence)
extractor := kg.NewExtractor(p, model, minConfidence)
extractor.SetUsageCapService(h.usageCaps)
return extractor
}
// RegisterRoutes registers all KG routes on the given mux.
+11 -6
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"strconv"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/store"
)
@@ -156,6 +157,10 @@ func (h *KnowledgeGraphHandler) handleTraverse(w http.ResponseWriter, r *http.Re
func (h *KnowledgeGraphHandler) handleExtract(w http.ResponseWriter, r *http.Request) {
locale := extractLocale(r)
agentID := r.PathValue("agentID")
callCtx := r.Context()
if parsedAgentID, err := uuid.Parse(agentID); err == nil {
callCtx = store.WithAgentID(callCtx, parsedAgentID)
}
var body struct {
Text string `json:"text"`
@@ -176,13 +181,13 @@ func (h *KnowledgeGraphHandler) handleExtract(w http.ResponseWriter, r *http.Req
return
}
extractor := h.NewExtractor(r.Context(), body.Provider, body.Model, body.MinConf)
extractor := h.NewExtractor(callCtx, body.Provider, body.Model, body.MinConf)
if extractor == nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidProviderOrModel)})
return
}
result, err := extractor.Extract(r.Context(), body.Text)
result, err := extractor.Extract(callCtx, body.Text)
if err != nil {
slog.Warn("kg.extract failed", "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
@@ -213,10 +218,10 @@ func (h *KnowledgeGraphHandler) handleExtract(w http.ResponseWriter, r *http.Req
}
writeJSON(w, http.StatusOK, map[string]any{
"entities": len(result.Entities),
"relations": len(result.Relations),
"dedup_merged": dedupMerged,
"dedup_flagged": dedupFlagged,
"entities": len(result.Entities),
"relations": len(result.Relations),
"dedup_merged": dedupMerged,
"dedup_flagged": dedupFlagged,
})
}
+7 -1
View File
@@ -11,6 +11,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/providerresolve"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// PendingMessagesHandler handles pending message HTTP endpoints.
@@ -22,6 +23,7 @@ type PendingMessagesHandler struct {
maxTokens int // max output tokens for LLM summarization (0 = use default)
cfgProvider string // config-level provider override (empty = resolve from agent)
cfgModel string // config-level model override (empty = resolve from agent)
usageCaps *usagecaps.Service
}
func NewPendingMessagesHandler(s store.PendingMessageStore, agentStore store.AgentStore, providerReg *providers.Registry) *PendingMessagesHandler {
@@ -40,6 +42,10 @@ func (h *PendingMessagesHandler) SetProviderModel(provider, model string) {
h.cfgModel = model
}
func (h *PendingMessagesHandler) SetUsageCapService(s *usagecaps.Service) {
h.usageCaps = s
}
func (h *PendingMessagesHandler) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("GET /v1/pending-messages", h.authMiddleware(h.handleListGroups))
mux.HandleFunc("GET /v1/pending-messages/messages", h.authMiddleware(h.handleListMessages))
@@ -148,7 +154,7 @@ func (h *PendingMessagesHandler) handleCompact(w http.ResponseWriter, r *http.Re
go func() {
ctx, cancel := context.WithTimeout(store.WithTenantID(context.Background(), tenantID), 180*time.Second)
defer cancel()
remaining, err := channels.CompactGroup(ctx, h.store, req.ChannelName, req.HistoryKey, provider, model, keepRecent, h.maxTokens)
remaining, err := channels.CompactGroup(ctx, h.store, req.ChannelName, req.HistoryKey, provider, model, keepRecent, h.maxTokens, h.usageCaps)
if err != nil {
slog.Warn("compact.failed", "channel", req.ChannelName, "key", req.HistoryKey, "error", err)
} else {
+10 -1
View File
@@ -16,6 +16,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// HandleVerifyProviderForTest invokes the verify handler directly without auth
@@ -126,8 +127,9 @@ func (h *ProvidersHandler) handleVerifyProvider(w http.ResponseWriter, r *http.R
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
ctx = store.WithTenantID(ctx, p.TenantID)
_, err = provider.Chat(ctx, providers.ChatRequest{
reqChat := providers.ChatRequest{
Messages: []providers.Message{
{Role: "user", Content: "hi"},
},
@@ -136,6 +138,13 @@ func (h *ProvidersHandler) handleVerifyProvider(w http.ResponseWriter, r *http.R
// Use a small but safe value — reasoning models need headroom beyond 1 token.
"max_tokens": 50,
},
}
_, err = h.usageCaps.Chat(ctx, provider, reqChat, usagecaps.ChatOptions{
TenantID: p.TenantID,
ProviderName: p.Name,
ModelID: req.Model,
Purpose: "provider-verify",
MaxOutputTokens: 50,
})
if err != nil {
writeJSON(w, http.StatusOK, map[string]any{"valid": false, "error": friendlyVerifyError(err)})
+6
View File
@@ -22,6 +22,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
@@ -39,6 +40,7 @@ type ProvidersHandler struct {
tracingStore store.TracingStore // optional: for provider-scoped pool activity
agents store.AgentCRUDStore // optional: for provider pool activity agent lookup
modelReg providers.ModelRegistry // optional: forward-compat model resolver for Anthropic
usageCaps *usagecaps.Service
}
// NewProvidersHandler creates a handler for provider management endpoints.
@@ -85,6 +87,10 @@ func (h *ProvidersHandler) SetModelRegistry(r providers.ModelRegistry) {
h.modelReg = r
}
func (h *ProvidersHandler) SetUsageCapService(s *usagecaps.Service) {
h.usageCaps = s
}
// resolveAPIBase returns the provider's api_base, falling back to config/env if empty.
// For Ollama/OllamaCloud providers, applies a safety-net normalization: if the stored
// value is missing the /v1 suffix (pre-existing record before write-time normalization),
+43 -99
View File
@@ -7,7 +7,6 @@ import (
"net/http"
"os/exec"
"regexp"
"sort"
"strings"
"github.com/google/uuid"
@@ -70,68 +69,13 @@ func (h *SecureCLIHandler) emitCacheInvalidate(key string) {
// envKeysFromDecryptedJSON returns sorted env variable names from plaintext env JSON (decrypted blob).
func envKeysFromDecryptedJSON(env []byte) []string {
empty := []string{}
if len(env) == 0 {
return empty
}
var m map[string]any
if err := json.Unmarshal(env, &m); err != nil {
return empty
}
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
return store.SecureCLIEnvKeys(env)
}
// mergeSecureCLIEnv merges incoming env from the UI with existing stored env.
// Incoming defines the full set of keys shown in the form: keys omitted were removed.
// Empty string means "keep existing value" for that key when it already exists.
func mergeSecureCLIEnv(existingJSON []byte, incoming map[string]any) (map[string]string, error) {
existing := map[string]string{}
if len(existingJSON) > 0 {
if err := json.Unmarshal(existingJSON, &existing); err != nil {
return nil, fmt.Errorf("parse existing env: %w", err)
}
}
out := make(map[string]string)
for k, v := range incoming {
if k == "" {
continue
}
sv, err := envValueAsString(v)
if err != nil {
return nil, fmt.Errorf("invalid environment variable value")
}
if sv != "" {
out[k] = sv
continue
}
if ev, ok := existing[k]; ok {
out[k] = ev
}
}
return out, nil
}
func envValueAsString(v any) (string, error) {
switch t := v.(type) {
case string:
return t, nil
case float64:
return fmt.Sprint(t), nil
case bool:
if t {
return "true", nil
}
return "false", nil
case nil:
return "", nil
default:
return "", fmt.Errorf("value must be a string")
}
func populateBinaryEnvResponse(b *store.SecureCLIBinary) {
b.EnvKeys = store.SecureCLIEnvKeys(b.EncryptedEnv)
b.Env = store.SanitizeSecureCLIEnvJSON(b.EncryptedEnv)
b.EncryptedEnv = nil
}
func (h *SecureCLIHandler) handleList(w http.ResponseWriter, r *http.Request) {
@@ -142,27 +86,26 @@ func (h *SecureCLIHandler) handleList(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgFailedToList, "CLI credentials")})
return
}
// Never send env values; only variable names for editing.
// Sensitive env values stay masked; value-kind entries are returned for editing.
for i := range result {
result[i].EnvKeys = envKeysFromDecryptedJSON(result[i].EncryptedEnv)
result[i].EncryptedEnv = nil
populateBinaryEnvResponse(&result[i])
}
writeJSON(w, http.StatusOK, map[string]any{"items": result})
}
// secureCLICreateRequest supports both preset-based and custom creation.
type secureCLICreateRequest struct {
Preset string `json:"preset,omitempty"` // auto-fill from preset
BinaryName string `json:"binary_name"`
BinaryPath *string `json:"binary_path,omitempty"`
Description string `json:"description"`
Env map[string]string `json:"env"` // plaintext env vars (encrypted by store)
DenyArgs json.RawMessage `json:"deny_args,omitempty"`
DenyVerbose json.RawMessage `json:"deny_verbose,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
Tips string `json:"tips,omitempty"`
IsGlobal *bool `json:"is_global,omitempty"`
Enabled bool `json:"enabled"`
Preset string `json:"preset,omitempty"` // auto-fill from preset
BinaryName string `json:"binary_name"`
BinaryPath *string `json:"binary_path,omitempty"`
Description string `json:"description"`
Env json.RawMessage `json:"env"` // plaintext env vars or env entry objects (encrypted by store)
DenyArgs json.RawMessage `json:"deny_args,omitempty"`
DenyVerbose json.RawMessage `json:"deny_verbose,omitempty"`
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
Tips string `json:"tips,omitempty"`
IsGlobal *bool `json:"is_global,omitempty"`
Enabled bool `json:"enabled"`
}
func (h *SecureCLIHandler) handleCreate(w http.ResponseWriter, r *http.Request) {
@@ -209,10 +152,14 @@ func (h *SecureCLIHandler) handleCreate(w http.ResponseWriter, r *http.Request)
return
}
// Serialize env as JSON bytes (store layer encrypts)
envJSON, err := json.Marshal(req.Env)
envEntries, err := store.ParseSecureCLIEnv(req.Env)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid env"})
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return
}
envJSON, err := store.SerializeSecureCLIEnv(envEntries)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return
}
@@ -240,7 +187,7 @@ func (h *SecureCLIHandler) handleCreate(w http.ResponseWriter, r *http.Request)
}
tools.ResetCredentialScrubValues() // clear stale scrub values
b.EncryptedEnv = nil // don't return credentials
populateBinaryEnvResponse(b)
emitAudit(h.msgBus, r, "secure_cli.created", "secure_cli", b.ID.String())
h.emitCacheInvalidate(b.ID.String())
writeJSON(w, http.StatusCreated, b)
@@ -260,8 +207,7 @@ func (h *SecureCLIHandler) handleGet(w http.ResponseWriter, r *http.Request) {
return
}
b.EnvKeys = envKeysFromDecryptedJSON(b.EncryptedEnv)
b.EncryptedEnv = nil // don't expose credential values
populateBinaryEnvResponse(b)
writeJSON(w, http.StatusOK, b)
}
@@ -293,25 +239,23 @@ func (h *SecureCLIHandler) handleUpdate(w http.ResponseWriter, r *http.Request)
// If env is updated, merge with stored env so empty values mean "keep existing secret".
if envVal, ok := updates["env"]; ok {
if envMap, isMap := envVal.(map[string]any); isMap {
cur, err := h.store.Get(r.Context(), id)
if err != nil {
writeJSON(w, http.StatusNotFound, map[string]string{"error": i18n.T(locale, i18n.MsgNotFound, "credential", id.String())})
return
}
merged, err := mergeSecureCLIEnv(cur.EncryptedEnv, envMap)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
return
}
envJSON, err := json.Marshal(merged)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid env"})
return
}
updates["encrypted_env"] = string(envJSON)
delete(updates, "env")
envJSON, err := json.Marshal(envVal)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return
}
cur, err := h.store.Get(r.Context(), id)
if err != nil {
writeJSON(w, http.StatusNotFound, map[string]string{"error": i18n.T(locale, i18n.MsgNotFound, "credential", id.String())})
return
}
merged, err := store.MergeSecureCLIEnv(cur.EncryptedEnv, envJSON)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return
}
updates["encrypted_env"] = string(merged)
delete(updates, "env")
}
if err := h.store.Update(r.Context(), id, updates); err != nil {
+38 -38
View File
@@ -68,7 +68,7 @@ func (h *SecureCLIGrantHandler) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("GET /v1/cli-credentials/{id}/agent-grants/{grantId}", auth(h.handleGet))
mux.HandleFunc("PUT /v1/cli-credentials/{id}/agent-grants/{grantId}", auth(h.handleUpdate))
mux.HandleFunc("DELETE /v1/cli-credentials/{id}/agent-grants/{grantId}", auth(h.handleDelete))
// POST (not GET) to prevent caching and satisfy CSRF semantics per Red Team C1.
// POST keeps revealed secret material out of URL/history and avoids query caching.
mux.HandleFunc("POST /v1/cli-credentials/{id}/agent-grants/{grantId}/env:reveal", auth(h.handleRevealEnv))
}
@@ -76,45 +76,41 @@ func (h *SecureCLIGrantHandler) RegisterRoutes(mux *http.ServeMux) {
// EnvVars is optional; plaintext values are encrypted by the store layer.
// Clients MUST NOT send encrypted_env — that field is never accepted from the wire.
type grantCreateRequest struct {
AgentID uuid.UUID `json:"agent_id"`
EnvVars map[string]string `json:"env_vars,omitempty"`
DenyArgs *json.RawMessage `json:"deny_args,omitempty"`
DenyVerbose *json.RawMessage `json:"deny_verbose,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
Tips *string `json:"tips,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
AgentID uuid.UUID `json:"agent_id"`
EnvVars json.RawMessage `json:"env_vars,omitempty"`
DenyArgs *json.RawMessage `json:"deny_args,omitempty"`
DenyVerbose *json.RawMessage `json:"deny_verbose,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
Tips *string `json:"tips,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
}
// populateGrantEnvFields sets EnvKeys (sorted) and EnvSet from the grant's decrypted env bytes.
// Plaintext values are never exposed — only key names.
// populateGrantEnvFields sets sorted key names, env presence, and sanitized entries.
func populateGrantEnvFields(g *store.SecureCLIAgentGrant) {
if len(g.EncryptedEnv) == 0 {
g.EnvKeys = []string{}
g.Env = nil
g.EnvSet = false
return
}
var m map[string]any
if err := json.Unmarshal(g.EncryptedEnv, &m); err != nil {
g.EnvKeys = []string{}
g.EnvSet = false
return
}
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
keys := store.SecureCLIEnvKeys(g.EncryptedEnv)
g.EnvKeys = keys
g.Env = store.SanitizeSecureCLIEnvJSON(g.EncryptedEnv)
g.EnvSet = len(keys) > 0
}
// validateAndSerializeEnvVars validates env keys/values via denylist and returns serialized JSON.
// Returns (nil, 400 error response written) on denial, (jsonBytes, nil) on success.
// Never logs env values or keys in error paths.
func validateAndSerializeEnvVars(w http.ResponseWriter, locale string, envVars map[string]string) ([]byte, bool) {
if len(envVars) == 0 {
b, _ := json.Marshal(envVars)
return b, true
func validateAndSerializeEnvVars(w http.ResponseWriter, locale string, raw json.RawMessage) ([]byte, bool) {
envEntries, err := store.ParseSecureCLIEnv(raw)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return nil, false
}
envVars := make(map[string]string, len(envEntries))
for key, entry := range envEntries {
envVars[key] = entry.Value
}
denied, valErr := crypto.ValidateGrantEnvVars(envVars)
if valErr != nil {
@@ -129,7 +125,7 @@ func validateAndSerializeEnvVars(w http.ResponseWriter, locale string, envVars m
})
return nil, false
}
b, err := json.Marshal(envVars)
b, err := store.SerializeSecureCLIEnv(envEntries)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, "serialization failed")})
return nil, false
@@ -249,7 +245,7 @@ func (h *SecureCLIGrantHandler) handleCreate(w http.ResponseWriter, r *http.Requ
envJSON, ok := validateAndSerializeEnvVars(w, locale, req.EnvVars)
if !ok {
// Grant was created but env validation failed; clean it up to avoid orphan row.
// Finding #13: log rollback-delete failures for ops visibility.
// Log rollback-delete failures so operators can clean up orphan rows.
if delErr := h.grants.Delete(r.Context(), g.ID); delErr != nil {
slog.Error("secure_cli_grants.create.rollback_delete",
"grant_id", g.ID,
@@ -261,7 +257,7 @@ func (h *SecureCLIGrantHandler) handleCreate(w http.ResponseWriter, r *http.Requ
}
if err := h.grants.UpdateGrantEnv(r.Context(), g.ID, envJSON); err != nil {
slog.Error("secure_cli_grants.create.set_env", "grant_id", g.ID, "error", err)
// Finding #13: log rollback-delete failures for ops visibility.
// Log rollback-delete failures so operators can clean up orphan rows.
if delErr := h.grants.Delete(r.Context(), g.ID); delErr != nil {
slog.Error("secure_cli_grants.create.rollback_delete",
"grant_id", g.ID,
@@ -323,7 +319,7 @@ func (h *SecureCLIGrantHandler) handleUpdate(w http.ResponseWriter, r *http.Requ
}
if allowedScalar[k] {
var decoded any
// Finding #3: return 400 on Unmarshal failure — silent discard means admin
// Return 400 on unmarshal failure; silent discard means admin
// thinks they applied a change (e.g. enabled: "false") but the grant is unchanged.
if err := json.Unmarshal(v, &decoded); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{
@@ -335,7 +331,7 @@ func (h *SecureCLIGrantHandler) handleUpdate(w http.ResponseWriter, r *http.Requ
}
}
// 3-state env_vars semantics: absent=skip, null=clear, {...}=replace.
// Finding #15: {} (empty map) is treated as clear — same as null.
// Empty map is treated as clear, same as null.
// TS type: absent | null | Record<string,string> — see ui/web/src/types/cli-credential.ts.
var envJSON []byte
envPresent := false
@@ -343,18 +339,22 @@ func (h *SecureCLIGrantHandler) handleUpdate(w http.ResponseWriter, r *http.Requ
envPresent = true
var envPtr *map[string]string
if string(envRaw) != "null" {
var m map[string]string
if err := json.Unmarshal(envRaw, &m); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, "env_vars must be a string map")})
envEntries, err := store.ParseSecureCLIEnv(envRaw)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, "env_vars must be a string map or env entries")})
return
}
m := make(map[string]string, len(envEntries))
for key, entry := range envEntries {
m[key] = entry.Value
}
envPtr = &m
}
// envPtr == nil → clear; envPtr != nil → replace.
// Note: envPtr pointing to an empty map ({}) is treated as clear (same as null) —
// envJSON stays nil and UpdateGrantEnv(nil) removes the override.
if envPtr != nil && len(*envPtr) > 0 {
j, ok := validateAndSerializeEnvVars(w, locale, *envPtr)
j, ok := validateAndSerializeEnvVars(w, locale, envRaw)
if !ok {
return
}
@@ -430,8 +430,8 @@ func (h *SecureCLIGrantHandler) handleRevealEnv(w http.ResponseWriter, r *http.R
locale := store.LocaleFromContext(ctx)
// Rate limit: 10 reveals/min per authenticated caller (context UserID).
// Finding #2: require non-empty UserID from authenticated context.
// If UserID is empty, the auth middleware failed to populate it — reject rather
// Require non-empty UserID from authenticated context.
// If UserID is empty, auth middleware failed to populate it — reject rather
// than fall back to a spoofable header or IP address.
callerID := store.UserIDFromContext(ctx)
if callerID == "" {
@@ -476,8 +476,8 @@ func (h *SecureCLIGrantHandler) handleRevealEnv(w http.ResponseWriter, r *http.R
writeJSON(w, http.StatusOK, map[string]any{"env_vars": map[string]string{}})
return
}
var envVars map[string]string
if err := json.Unmarshal(g.EncryptedEnv, &envVars); err != nil {
envVars, err := store.FlattenSecureCLIEnv(g.EncryptedEnv)
if err != nil {
slog.Error("secure_cli_grants.reveal.parse", "grant_id", g.ID, "error", err)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "parse grant env")})
return
+50 -2
View File
@@ -3,6 +3,7 @@ package http
import (
"context"
"database/sql"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
@@ -59,8 +60,16 @@ func (s *fakeSecureCLIGrantStore) Delete(context.Context, uuid.UUID) error {
return nil
}
func (s *fakeSecureCLIGrantStore) ListByBinary(context.Context, uuid.UUID) ([]store.SecureCLIAgentGrant, error) {
return nil, nil
func (s *fakeSecureCLIGrantStore) ListByBinary(_ context.Context, binaryID uuid.UUID) ([]store.SecureCLIAgentGrant, error) {
grants := make([]store.SecureCLIAgentGrant, 0, len(s.grants))
for _, grant := range s.grants {
if grant == nil || grant.BinaryID != binaryID {
continue
}
cp := *grant
grants = append(grants, cp)
}
return grants, nil
}
func (s *fakeSecureCLIGrantStore) ListByAgent(context.Context, uuid.UUID) ([]store.SecureCLIAgentGrant, error) {
@@ -202,3 +211,42 @@ func TestSecureCLIGrantUpdateRejectsInvalidEnvVarsBeforeScalarUpdate(t *testing.
t.Fatal("invalid env_vars request must not persist scalar grant updates")
}
}
func TestSecureCLIGrantGetSanitizesMixedEnv(t *testing.T) {
binaryID := uuid.New()
grantID := uuid.New()
fake := &fakeSecureCLIGrantStore{
grants: map[uuid.UUID]*store.SecureCLIAgentGrant{
grantID: {
BaseModel: store.BaseModel{ID: grantID},
BinaryID: binaryID,
AgentID: uuid.New(),
Enabled: true,
EncryptedEnv: []byte(`{"TOKEN":"secret-token","PUBLIC_BASE_URL":{"kind":"value","value":"https://goclaw.sh"}}`),
},
},
}
h := NewSecureCLIGrantHandler(fake, nil, nil)
rr, req := requestWithGrantPath(http.MethodGet, nil, binaryID, grantID)
h.handleGet(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String())
}
if strings.Contains(rr.Body.String(), "secret-token") {
t.Fatalf("sensitive grant env leaked in response: %s", rr.Body.String())
}
var got struct {
Env map[string]store.SecureCLIEnvResponseEntry `json:"env"`
}
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if !got.Env["TOKEN"].Masked || got.Env["TOKEN"].Value != nil {
t.Fatalf("TOKEN not masked: %#v", got.Env["TOKEN"])
}
if got.Env["PUBLIC_BASE_URL"].Value == nil || *got.Env["PUBLIC_BASE_URL"].Value != "https://goclaw.sh" {
t.Fatalf("PUBLIC_BASE_URL not returned: %#v", got.Env["PUBLIC_BASE_URL"])
}
}
@@ -0,0 +1,129 @@
package http
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/store"
)
type fakeSecureCLIStore struct {
binary *store.SecureCLIBinary
user *store.SecureCLIUserCredential
}
func (s *fakeSecureCLIStore) Create(context.Context, *store.SecureCLIBinary) error { return nil }
func (s *fakeSecureCLIStore) Get(context.Context, uuid.UUID) (*store.SecureCLIBinary, error) {
cp := *s.binary
return &cp, nil
}
func (s *fakeSecureCLIStore) Update(context.Context, uuid.UUID, map[string]any) error { return nil }
func (s *fakeSecureCLIStore) Delete(context.Context, uuid.UUID) error { return nil }
func (s *fakeSecureCLIStore) List(context.Context) ([]store.SecureCLIBinary, error) {
cp := *s.binary
return []store.SecureCLIBinary{cp}, nil
}
func (s *fakeSecureCLIStore) LookupByBinary(context.Context, string, *uuid.UUID, string) (*store.SecureCLIBinary, error) {
return nil, nil
}
func (s *fakeSecureCLIStore) ListEnabled(context.Context) ([]store.SecureCLIBinary, error) {
return nil, nil
}
func (s *fakeSecureCLIStore) ListForAgent(context.Context, uuid.UUID) ([]store.SecureCLIBinary, error) {
return nil, nil
}
func (s *fakeSecureCLIStore) IsRegisteredBinary(context.Context, string) (bool, error) {
return false, nil
}
func (s *fakeSecureCLIStore) GetUserCredentials(context.Context, uuid.UUID, string) (*store.SecureCLIUserCredential, error) {
cp := *s.user
return &cp, nil
}
func (s *fakeSecureCLIStore) SetUserCredentials(context.Context, uuid.UUID, string, []byte) error {
return nil
}
func (s *fakeSecureCLIStore) DeleteUserCredentials(context.Context, uuid.UUID, string) error {
return nil
}
func (s *fakeSecureCLIStore) ListUserCredentials(context.Context, uuid.UUID) ([]store.SecureCLIUserCredential, error) {
cp := *s.user
return []store.SecureCLIUserCredential{cp}, nil
}
func TestSecureCLIGetSanitizesMixedEnv(t *testing.T) {
id := uuid.New()
h := NewSecureCLIHandler(&fakeSecureCLIStore{
binary: &store.SecureCLIBinary{
BaseModel: store.BaseModel{ID: id},
BinaryName: "gh",
EncryptedEnv: []byte(`{"TOKEN":"secret-token","PUBLIC_BASE_URL":{"kind":"value","value":"https://goclaw.sh"}}`),
},
}, nil)
req := httptest.NewRequest(http.MethodGet, "/v1/cli-credentials/"+id.String(), nil)
req.SetPathValue("id", id.String())
rec := httptest.NewRecorder()
h.handleGet(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "secret-token") {
t.Fatalf("sensitive env leaked in response: %s", rec.Body.String())
}
var got struct {
Env map[string]store.SecureCLIEnvResponseEntry `json:"env"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if !got.Env["TOKEN"].Masked || got.Env["TOKEN"].Value != nil {
t.Fatalf("TOKEN not masked: %#v", got.Env["TOKEN"])
}
if got.Env["PUBLIC_BASE_URL"].Value == nil || *got.Env["PUBLIC_BASE_URL"].Value != "https://goclaw.sh" {
t.Fatalf("value env not returned: %#v", got.Env["PUBLIC_BASE_URL"])
}
}
func TestSecureCLIUserCredentialsGetDoesNotReturnLegacySensitiveRaw(t *testing.T) {
binaryID := uuid.New()
h := NewSecureCLIHandler(&fakeSecureCLIStore{
user: &store.SecureCLIUserCredential{
ID: uuid.New(),
BinaryID: binaryID,
UserID: "user-1",
EncryptedEnv: []byte(`{"TOKEN":"secret-token","REGION":{"kind":"value","value":"asia-southeast1"}}`),
},
}, nil)
req := httptest.NewRequest(http.MethodGet, "/v1/cli-credentials/"+binaryID.String()+"/user-credentials/user-1", nil)
req.SetPathValue("id", binaryID.String())
req.SetPathValue("userId", "user-1")
rec := httptest.NewRecorder()
h.handleGetUserCredentials(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "secret-token") {
t.Fatalf("legacy sensitive env leaked in response: %s", rec.Body.String())
}
var got struct {
Env map[string]store.SecureCLIEnvResponseEntry `json:"env"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if !got.Env["TOKEN"].Masked || got.Env["TOKEN"].Value != nil {
t.Fatalf("TOKEN not masked: %#v", got.Env["TOKEN"])
}
if got.Env["REGION"].Value == nil || *got.Env["REGION"].Value != "asia-southeast1" {
t.Fatalf("REGION not returned: %#v", got.Env["REGION"])
}
}
+24 -18
View File
@@ -25,13 +25,14 @@ func (h *SecureCLIHandler) handleListUserCredentials(w http.ResponseWriter, r *h
}
// Return without env values for listing (names only + timestamps)
type entry struct {
ID uuid.UUID `json:"id"`
BinaryID uuid.UUID `json:"binary_id"`
UserID string `json:"user_id"`
HasEnv bool `json:"has_env"`
EnvKeys []string `json:"env_keys,omitempty"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
ID uuid.UUID `json:"id"`
BinaryID uuid.UUID `json:"binary_id"`
UserID string `json:"user_id"`
HasEnv bool `json:"has_env"`
EnvKeys []string `json:"env_keys,omitempty"`
Env map[string]store.SecureCLIEnvResponseEntry `json:"env,omitempty"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
entries := make([]entry, 0, len(creds))
for _, c := range creds {
@@ -42,6 +43,7 @@ func (h *SecureCLIHandler) handleListUserCredentials(w http.ResponseWriter, r *h
UserID: c.UserID,
HasEnv: len(c.EncryptedEnv) > 0,
EnvKeys: envKeys,
Env: store.SanitizeSecureCLIEnvJSON(c.EncryptedEnv),
CreatedAt: c.CreatedAt,
UpdatedAt: c.UpdatedAt,
})
@@ -73,14 +75,9 @@ func (h *SecureCLIHandler) handleGetUserCredentials(w http.ResponseWriter, r *ht
return
}
// Return decrypted env as JSON object (admin-only endpoint)
var envObj any
if len(cred.EncryptedEnv) > 0 {
_ = json.Unmarshal(cred.EncryptedEnv, &envObj)
}
writeJSON(w, http.StatusOK, map[string]any{
"user_id": cred.UserID,
"env": envObj,
"env": store.SanitizeSecureCLIEnvJSON(cred.EncryptedEnv),
})
}
@@ -108,14 +105,23 @@ func (h *SecureCLIHandler) handleSetUserCredentials(w http.ResponseWriter, r *ht
return
}
// Validate env is a JSON object
var envCheck map[string]string
if err := json.Unmarshal(body.Env, &envCheck); err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "env must be a JSON object with string values"})
existing, err := h.store.GetUserCredentials(r.Context(), binaryID, userID)
if err != nil {
locale := store.LocaleFromContext(r.Context())
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, err.Error())})
return
}
var existingEnv []byte
if existing != nil {
existingEnv = existing.EncryptedEnv
}
envJSON, err := store.MergeSecureCLIEnv(existingEnv, body.Env)
if err != nil {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
return
}
if err := h.store.SetUserCredentials(r.Context(), binaryID, userID, body.Env); err != nil {
if err := h.store.SetUserCredentials(r.Context(), binaryID, userID, envJSON); err != nil {
locale := store.LocaleFromContext(r.Context())
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, err.Error())})
return
+3 -3
View File
@@ -21,8 +21,6 @@ import (
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
const maxSkillUploadSize = 20 << 20 // 20 MB
var (
aggregateInstallDeps = skills.AggregateMissingDeps
installManagedDeps = skills.InstallDeps
@@ -40,11 +38,13 @@ type SkillsHandler struct {
tenantStore store.TenantStore
db *sql.DB // for export/import direct queries
uploadLocks sync.Map // per-slug mutex; bounded by validated slug set, entries are tiny (*sync.Mutex)
uploadLimitCfg config.SkillsConfig
systemConfigs store.SystemConfigStore
}
// NewSkillsHandler creates a handler for skill management endpoints.
func NewSkillsHandler(skills store.SkillManageStore, baseDir, dataDir, bundledDir string, msgBus *bus.MessageBus, tenantCfgStore store.SkillTenantConfigStore, tenantStore store.TenantStore) *SkillsHandler {
return &SkillsHandler{skills: skills, baseDir: baseDir, dataDir: dataDir, bundledDir: bundledDir, msgBus: msgBus, tenantCfgStore: tenantCfgStore, tenantStore: tenantStore}
return &SkillsHandler{skills: skills, baseDir: baseDir, dataDir: dataDir, bundledDir: bundledDir, msgBus: msgBus, tenantCfgStore: tenantCfgStore, tenantStore: tenantStore, uploadLimitCfg: config.SkillsConfig{MaxUploadSizeMB: config.DefaultSkillMaxUploadSizeMB}}
}
// tenantSkillsDir returns the skills-store directory scoped to the requesting tenant.
-15
View File
@@ -1,8 +1,6 @@
package http
import (
"archive/zip"
"io"
"log/slog"
"net/http"
@@ -225,16 +223,3 @@ func (h *SkillsHandler) handleRevokeUser(w http.ResponseWriter, r *http.Request)
}
// --- Helpers ---
func readZipFile(f *zip.File) (string, error) {
rc, err := f.Open()
if err != nil {
return "", err
}
defer rc.Close()
data, err := io.ReadAll(rc)
if err != nil {
return "", err
}
return string(data), nil
}
+14 -8
View File
@@ -5,6 +5,7 @@ import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
@@ -43,7 +44,8 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
return
}
r.Body = http.MaxBytesReader(w, r.Body, maxSkillUploadSize)
preParseLimitMB := h.resolvePreParseUploadLimitMB(r.Context())
r.Body = http.MaxBytesReader(w, r.Body, skillUploadBodyBytes(preParseLimitMB))
file, header, err := r.FormFile("file")
if err != nil {
@@ -70,6 +72,11 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
size, err := io.Copy(tmp, file)
if err != nil {
tmp.Close()
var maxBytesErr *http.MaxBytesError
if errors.As(err, &maxBytesErr) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, skillUploadTooLargeMessage(maxBytesErr.Limit, preParseLimitMB))})
return
}
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "failed to save upload")})
return
}
@@ -141,6 +148,11 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
}
name, description, slug, frontmatter := skills.ParseSkillFrontmatter(skillContent)
uploadLimitMB := h.resolveSkillUploadLimitMB(r.Context(), frontmatter)
if size > skillUploadLimitBytes(uploadLimitMB) {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, skillUploadTooLargeMessage(size, uploadLimitMB))})
return
}
if name == "" {
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgRequired, "name in SKILL.md frontmatter")})
return
@@ -239,17 +251,11 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "failed to create skill file directory")})
return
}
data, err := readZipFile(f)
if err != nil {
if err := copyZipFileToPath(f, destPath); err != nil {
os.RemoveAll(destDir)
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, "failed to read ZIP entry")})
return
}
if err := os.WriteFile(destPath, []byte(data), 0644); err != nil {
os.RemoveAll(destDir)
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "failed to write skill files")})
return
}
if cleanName == "SKILL.md" {
wroteSkillMD = true
}
+88
View File
@@ -0,0 +1,88 @@
package http
import (
"context"
"fmt"
"strconv"
"strings"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/store"
)
const (
skillUploadMaxSizeConfigKey = config.SkillMaxUploadSizeSystemConfigKey
skillUploadMultipartOverheadBytes = int64(1 << 20)
)
func (h *SkillsHandler) SetUploadLimitConfig(cfg config.SkillsConfig) {
h.uploadLimitCfg = cfg
}
func (h *SkillsHandler) SetSystemConfigStore(s store.SystemConfigStore) {
h.systemConfigs = s
}
func (h *SkillsHandler) resolveSkillUploadLimitMB(ctx context.Context, frontmatter map[string]string) int {
if mb, ok := h.resolveTenantSkillUploadLimitMB(ctx); ok {
return mb
}
if mb, ok := parseSkillUploadLimitMB(frontmatter["max_upload_size_mb"]); ok {
return config.ClampSkillMaxUploadSizeMB(mb)
}
return h.uploadLimitCfg.EffectiveMaxUploadSizeMB()
}
func (h *SkillsHandler) resolvePreParseUploadLimitMB(ctx context.Context) int {
if mb, ok := h.resolveTenantSkillUploadLimitMB(ctx); ok {
return mb
}
return config.MaxSkillMaxUploadSizeMB
}
func (h *SkillsHandler) resolveTenantSkillUploadLimitMB(ctx context.Context) (int, bool) {
if h.systemConfigs == nil {
return 0, false
}
raw, err := h.systemConfigs.Get(ctx, skillUploadMaxSizeConfigKey)
if err != nil {
return 0, false
}
mb, ok := parseSkillUploadLimitMB(raw)
if !ok {
return 0, false
}
return config.ClampSkillMaxUploadSizeMB(mb), true
}
func parseSkillUploadLimitMB(raw string) (int, bool) {
raw = strings.TrimSpace(raw)
if raw == "" {
return 0, false
}
mb, err := strconv.Atoi(raw)
if err != nil {
return 0, false
}
return mb, true
}
func skillUploadLimitBytes(limitMB int) int64 {
return int64(config.ClampSkillMaxUploadSizeMB(limitMB)) << 20
}
func skillUploadBodyBytes(limitMB int) int64 {
return skillUploadLimitBytes(limitMB) + skillUploadMultipartOverheadBytes
}
func skillUploadTooLargeMessage(size int64, limitMB int) string {
return fmt.Sprintf("skill ZIP size %s exceeds %d MB limit", formatUploadBytes(size), limitMB)
}
func formatUploadBytes(size int64) string {
const mb = int64(1 << 20)
if size >= mb {
return fmt.Sprintf("%.1f MB", float64(size)/float64(mb))
}
return fmt.Sprintf("%d bytes", size)
}
+126
View File
@@ -12,11 +12,13 @@ import (
"net/http/httptest"
"path/filepath"
"reflect"
"strings"
"testing"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/config"
"github.com/nextlevelbuilder/goclaw/internal/skills"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
@@ -670,6 +672,65 @@ func TestHandleUpload_ReturnsGrantErrors(t *testing.T) {
}
}
func TestResolveSkillUploadLimitMBPrecedenceAndClamp(t *testing.T) {
ctx := store.WithTenantID(context.Background(), store.MasterTenantID)
handler, _, _, _ := newTestUploadHandler(t)
handler.SetUploadLimitConfig(config.SkillsConfig{MaxUploadSizeMB: 30})
if got := handler.resolveSkillUploadLimitMB(ctx, nil); got != 30 {
t.Fatalf("global limit = %d, want 30", got)
}
if got := handler.resolveSkillUploadLimitMB(ctx, map[string]string{"max_upload_size_mb": "100"}); got != 100 {
t.Fatalf("frontmatter limit = %d, want 100", got)
}
handler.SetSystemConfigStore(&skillUploadSystemConfigStore{data: map[string]string{skillUploadMaxSizeConfigKey: "40"}})
if got := handler.resolveSkillUploadLimitMB(ctx, map[string]string{"max_upload_size_mb": "100"}); got != 40 {
t.Fatalf("tenant limit = %d, want 40", got)
}
handler.SetSystemConfigStore(&skillUploadSystemConfigStore{data: map[string]string{skillUploadMaxSizeConfigKey: "999"}})
if got := handler.resolveSkillUploadLimitMB(ctx, nil); got != config.MaxSkillMaxUploadSizeMB {
t.Fatalf("clamped tenant limit = %d, want %d", got, config.MaxSkillMaxUploadSizeMB)
}
}
func TestUploadRejectsZipAboveConfiguredLimit(t *testing.T) {
handler, _, ctx, _ := newTestUploadHandler(t)
handler.SetUploadLimitConfig(config.SkillsConfig{MaxUploadSizeMB: 1})
req := newZipUploadRequestWithBinary(t, ctx, map[string][]byte{
"SKILL.md": []byte(skillMarkdown("Large Skill", "large-skill")),
"large.bin": bytes.Repeat([]byte("x"), (1<<20)+1),
})
w := httptest.NewRecorder()
handler.handleUpload(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
}
if !strings.Contains(w.Body.String(), "exceeds") || !strings.Contains(w.Body.String(), "1 MB") {
t.Fatalf("body = %s, want configured upload limit error", w.Body.String())
}
}
func TestUploadAllowsFrontmatterLimitAboveGlobalDefault(t *testing.T) {
handler, _, ctx, _ := newTestUploadHandler(t)
handler.SetUploadLimitConfig(config.SkillsConfig{MaxUploadSizeMB: 1})
skillMD := "---\nname: Video Skill\nslug: video-skill\nmax_upload_size_mb: 2\n---\nSkill body\n"
req := newZipUploadRequestWithBinary(t, ctx, map[string][]byte{
"SKILL.md": []byte(skillMD),
"large.bin": bytes.Repeat([]byte("x"), (1<<20)+(128<<10)),
})
w := httptest.NewRecorder()
handler.handleUpload(w, req)
if w.Code != http.StatusCreated {
t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
}
}
func newTestUploadHandler(t *testing.T) (*SkillsHandler, *skillManageStoreStub, context.Context, string) {
t.Helper()
@@ -738,6 +799,43 @@ func newZipUploadRequestWithManagers(t *testing.T, ctx context.Context, files ma
return req.WithContext(ctx)
}
func newZipUploadRequestWithBinary(t *testing.T, ctx context.Context, files map[string][]byte) *http.Request {
t.Helper()
var zipBuf bytes.Buffer
zw := zip.NewWriter(&zipBuf)
for name, content := range files {
header := &zip.FileHeader{Name: name, Method: zip.Store}
w, err := zw.CreateHeader(header)
if err != nil {
t.Fatalf("zip create %s: %v", name, err)
}
if _, err := w.Write(content); err != nil {
t.Fatalf("zip write %s: %v", name, err)
}
}
if err := zw.Close(); err != nil {
t.Fatalf("zip close: %v", err)
}
var body bytes.Buffer
mw := multipart.NewWriter(&body)
part, err := mw.CreateFormFile("file", "skill.zip")
if err != nil {
t.Fatalf("multipart file: %v", err)
}
if _, err := part.Write(zipBuf.Bytes()); err != nil {
t.Fatalf("multipart write: %v", err)
}
if err := mw.Close(); err != nil {
t.Fatalf("multipart close: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/v1/skills/upload", &body)
req.Header.Set("Content-Type", mw.FormDataContentType())
return req.WithContext(ctx)
}
func newUploadManagerIDsFormRequest(t *testing.T, raw string) *http.Request {
t.Helper()
@@ -770,6 +868,34 @@ type skillManageStoreStub struct {
lastUpdates map[uuid.UUID]map[string]any
}
type skillUploadSystemConfigStore struct {
data map[string]string
}
func (s *skillUploadSystemConfigStore) Get(_ context.Context, key string) (string, error) {
if v, ok := s.data[key]; ok {
return v, nil
}
return "", errors.New("not found")
}
func (s *skillUploadSystemConfigStore) Set(_ context.Context, key, value string) error {
if s.data == nil {
s.data = map[string]string{}
}
s.data[key] = value
return nil
}
func (s *skillUploadSystemConfigStore) Delete(_ context.Context, key string) error {
delete(s.data, key)
return nil
}
func (s *skillUploadSystemConfigStore) List(_ context.Context) (map[string]string, error) {
return s.data, nil
}
type skillGrantCall struct {
SkillID uuid.UUID
AgentID uuid.UUID
+46
View File
@@ -0,0 +1,46 @@
package http
import (
"archive/zip"
"errors"
"io"
"os"
)
const maxSkillMarkdownBytes = 100 << 10
var errSkillMarkdownTooLarge = errors.New("SKILL.md exceeds 100 KB limit")
func readZipFile(f *zip.File) (string, error) {
rc, err := f.Open()
if err != nil {
return "", err
}
defer rc.Close()
data, err := io.ReadAll(io.LimitReader(rc, maxSkillMarkdownBytes+1))
if err != nil {
return "", err
}
if len(data) > maxSkillMarkdownBytes {
return "", errSkillMarkdownTooLarge
}
return string(data), nil
}
func copyZipFileToPath(f *zip.File, destPath string) error {
rc, err := f.Open()
if err != nil {
return err
}
defer rc.Close()
dst, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, rc)
return err
}
+5 -1
View File
@@ -12,6 +12,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
)
// Summoning event type constants.
@@ -48,14 +49,16 @@ type AgentSummoner struct {
agents store.AgentStore
providerReg *providers.Registry
msgBus *bus.MessageBus
usageCaps *usagecaps.Service
}
// NewAgentSummoner creates a summoner backed by the given stores and provider registry.
func NewAgentSummoner(agents store.AgentStore, providerReg *providers.Registry, msgBus *bus.MessageBus) *AgentSummoner {
func NewAgentSummoner(agents store.AgentStore, providerReg *providers.Registry, msgBus *bus.MessageBus, usageCaps *usagecaps.Service) *AgentSummoner {
return &AgentSummoner{
agents: agents,
providerReg: providerReg,
msgBus: msgBus,
usageCaps: usageCaps,
}
}
@@ -73,6 +76,7 @@ const singleCallTimeout = 300 * time.Second
func (s *AgentSummoner) SummonAgent(agentID uuid.UUID, tenantID uuid.UUID, providerName, model, description string) {
ctx, cancel := context.WithTimeout(store.WithTenantID(context.Background(), tenantID), 600*time.Second)
defer cancel()
ctx = store.WithAgentID(ctx, agentID)
s.ensureBackfillFiles(ctx, agentID)
s.emitEvent(agentID, tenantID, SummonEventStarted, "", "")
+9 -1
View File
@@ -14,6 +14,7 @@ import (
"github.com/nextlevelbuilder/goclaw/internal/bus"
"github.com/nextlevelbuilder/goclaw/internal/providers"
"github.com/nextlevelbuilder/goclaw/internal/store"
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
)
@@ -23,6 +24,7 @@ import (
func (s *AgentSummoner) RegenerateAgent(agentID uuid.UUID, tenantID uuid.UUID, providerName, model, editPrompt string) {
ctx, cancel := context.WithTimeout(store.WithTenantID(context.Background(), tenantID), 300*time.Second)
defer cancel()
ctx = store.WithAgentID(ctx, agentID)
s.ensureBackfillFiles(ctx, agentID)
@@ -121,7 +123,7 @@ func (s *AgentSummoner) generateFiles(ctx context.Context, providerName, model,
slog.Info("summoning: calling LLM", "provider", providerName, "model", model, "prompt_len", len(prompt))
resp, err := provider.Chat(ctx, providers.ChatRequest{
req := providers.ChatRequest{
Messages: []providers.Message{
{Role: "system", Content: "You are a file generator. Output ONLY the requested XML-tagged files. No extra commentary."},
{Role: "user", Content: prompt},
@@ -133,6 +135,12 @@ func (s *AgentSummoner) generateFiles(ctx context.Context, providerName, model,
providers.OptSessionKey: summonSessionKey,
providers.OptDisableTools: true,
},
}
resp, err := s.usageCaps.Chat(ctx, provider, req, usagecaps.ChatOptions{
ProviderName: providerName,
ModelID: model,
Purpose: "agent-summoner",
MaxOutputTokens: 8192,
})
if err != nil {
return nil, fmt.Errorf("%s: %w", providerName, err)
+397
View File
@@ -0,0 +1,397 @@
package http
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"strconv"
"strings"
"time"
"github.com/google/uuid"
"github.com/nextlevelbuilder/goclaw/internal/i18n"
"github.com/nextlevelbuilder/goclaw/internal/permissions"
"github.com/nextlevelbuilder/goclaw/internal/store"
"github.com/nextlevelbuilder/goclaw/internal/usage/pricing"
)
type UsageCapsHandler struct {
store store.UsageCapStore
tenants store.TenantStore
client *http.Client
}
func NewUsageCapsHandler(s store.UsageCapStore, tenants store.TenantStore) *UsageCapsHandler {
return &UsageCapsHandler{store: s, tenants: tenants, client: &http.Client{Timeout: 30 * time.Second}}
}
func (h *UsageCapsHandler) RegisterRoutes(mux *http.ServeMux) {
mux.HandleFunc("GET /v1/usage-caps/policies", h.auth(h.handleListPolicies))
mux.HandleFunc("POST /v1/usage-caps/policies", h.writeAuth(h.handleCreatePolicy))
mux.HandleFunc("PATCH /v1/usage-caps/policies/{id}", h.writeAuth(h.handleUpdatePolicy))
mux.HandleFunc("DELETE /v1/usage-caps/policies/{id}", h.writeAuth(h.handleDeletePolicy))
mux.HandleFunc("GET /v1/usage-caps/utilization", h.auth(h.handleUtilization))
mux.HandleFunc("GET /v1/usage-caps/events", h.auth(h.handleEvents))
mux.HandleFunc("POST /v1/model-pricing/sync-openrouter", h.masterAuth(h.handleSyncOpenRouter))
mux.HandleFunc("GET /v1/model-pricing", h.auth(h.handleListPricing))
mux.HandleFunc("PUT /v1/model-pricing/overrides", h.writeAuth(h.handlePutOverride))
mux.HandleFunc("GET /v1/model-pricing/overrides", h.auth(h.handleListOverrides))
mux.HandleFunc("DELETE /v1/model-pricing/overrides/{id}", h.writeAuth(h.handleDeleteOverride))
}
func (h *UsageCapsHandler) auth(next http.HandlerFunc) http.HandlerFunc {
return requireAuth(permissions.RoleAdmin, next)
}
func (h *UsageCapsHandler) writeAuth(next http.HandlerFunc) http.HandlerFunc {
return h.auth(func(w http.ResponseWriter, r *http.Request) {
if !requireTenantAdmin(w, r, h.tenants) {
return
}
next(w, r)
})
}
func (h *UsageCapsHandler) masterAuth(next http.HandlerFunc) http.HandlerFunc {
return h.auth(func(w http.ResponseWriter, r *http.Request) {
if !requireMasterScope(w, r) {
return
}
next(w, r)
})
}
func writeUsageCapError(w http.ResponseWriter, r *http.Request, status int, key string, args ...any) {
writeJSON(w, status, map[string]string{"error": i18n.T(store.LocaleFromContext(r.Context()), key, args...)})
}
func (h *UsageCapsHandler) handleListPolicies(w http.ResponseWriter, r *http.Request) {
scope := store.UsageCapScope{TenantID: tenantIDOrMaster(r)}
policies, err := h.store.ListUsageCapPolicies(r.Context(), scope, true)
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsListPoliciesFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"policies": policies})
}
func (h *UsageCapsHandler) handleCreatePolicy(w http.ResponseWriter, r *http.Request) {
var body policyBody
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidJSON)
return
}
p, err := body.toPolicy(tenantIDOrMaster(r))
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidRequest, err.Error())
return
}
if err := h.store.CreateUsageCapPolicy(r.Context(), &p); err != nil {
slog.Warn("usage_caps.create_policy_failed", "error", err)
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsageCapPolicyValidationFailed)
return
}
writeJSON(w, http.StatusCreated, p)
}
func (h *UsageCapsHandler) handleUpdatePolicy(w http.ResponseWriter, r *http.Request) {
id, err := uuid.Parse(r.PathValue("id"))
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "policy")
return
}
bodyBytes, err := io.ReadAll(r.Body)
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidJSON)
return
}
patch, err := policyPatchFromBody(bodyBytes)
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidRequest, err.Error())
return
}
p, err := h.store.UpdateUsageCapPolicy(r.Context(), tenantIDOrMaster(r), id, patch)
if err != nil {
if errors.Is(err, store.ErrUsageCapPolicyManaged) {
writeUsageCapError(w, r, http.StatusConflict, i18n.MsgUsageCapPolicyManaged)
return
}
slog.Warn("usage_caps.update_policy_failed", "error", err)
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsageCapPolicyValidationFailed)
return
}
writeJSON(w, http.StatusOK, p)
}
func (h *UsageCapsHandler) handleDeletePolicy(w http.ResponseWriter, r *http.Request) {
id, err := uuid.Parse(r.PathValue("id"))
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "policy")
return
}
if err := h.store.DeleteUsageCapPolicy(r.Context(), tenantIDOrMaster(r), id); err != nil {
if errors.Is(err, store.ErrUsageCapPolicyManaged) {
writeUsageCapError(w, r, http.StatusConflict, i18n.MsgUsageCapPolicyManaged)
return
}
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsDeletePolicyFailed)
return
}
w.WriteHeader(http.StatusNoContent)
}
func (h *UsageCapsHandler) handleUtilization(w http.ResponseWriter, r *http.Request) {
rows, err := h.store.ListUsageCapUtilization(r.Context(), tenantIDOrMaster(r))
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsUtilizationFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"rows": rows})
}
func (h *UsageCapsHandler) handleEvents(w http.ResponseWriter, r *http.Request) {
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
events, err := h.store.ListUsageCapEvents(r.Context(), tenantIDOrMaster(r), limit)
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsEventsFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"events": events})
}
func (h *UsageCapsHandler) handleSyncOpenRouter(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
defer cancel()
entries, err := pricing.FetchOpenRouterCatalog(ctx, h.client)
if err != nil {
slog.Warn("usage_pricing.openrouter_sync", "error", err)
writeUsageCapError(w, r, http.StatusBadGateway, i18n.MsgUsagePricingSyncOpenRouterFailed, err.Error())
return
}
count, err := h.store.UpsertPricingCatalog(r.Context(), entries)
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingStoreCatalogFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"count": count})
}
func (h *UsageCapsHandler) handleListPricing(w http.ResponseWriter, r *http.Request) {
rows, err := h.store.ListPricingCatalog(r.Context(), store.UsagePricingQuery{
ModelID: r.URL.Query().Get("model"),
Limit: queryInt(r, "limit", 100),
})
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingListFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"models": rows})
}
func (h *UsageCapsHandler) handlePutOverride(w http.ResponseWriter, r *http.Request) {
var body overrideBody
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidJSON)
return
}
providerID, err := uuid.Parse(body.ProviderID)
if err != nil || body.ModelID == "" {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsagePricingProviderModelRequired)
return
}
o := &store.UsagePricingOverride{
TenantID: tenantIDOrMaster(r), ProviderID: providerID,
ProviderType: body.ProviderType, ModelID: body.ModelID,
Pricing: body.Pricing, Enabled: body.Enabled == nil || *body.Enabled,
}
if err := h.store.PutPricingOverride(r.Context(), o); err != nil {
slog.Warn("usage_pricing.put_override_failed", "error", err)
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsagePricingOverrideValidationFailed)
return
}
writeJSON(w, http.StatusOK, o)
}
func (h *UsageCapsHandler) handleListOverrides(w http.ResponseWriter, r *http.Request) {
var providerID uuid.UUID
if raw := r.URL.Query().Get("provider_id"); raw != "" {
var err error
providerID, err = uuid.Parse(raw)
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "provider")
return
}
}
rows, err := h.store.ListPricingOverrides(r.Context(), store.UsagePricingQuery{TenantID: tenantIDOrMaster(r), ProviderID: providerID})
if err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingListOverridesFailed)
return
}
writeJSON(w, http.StatusOK, map[string]any{"overrides": rows})
}
func (h *UsageCapsHandler) handleDeleteOverride(w http.ResponseWriter, r *http.Request) {
id, err := uuid.Parse(r.PathValue("id"))
if err != nil {
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "override")
return
}
if err := h.store.DeletePricingOverride(r.Context(), tenantIDOrMaster(r), id); err != nil {
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingDeleteOverrideFailed)
return
}
w.WriteHeader(http.StatusNoContent)
}
type policyBody struct {
AgentID *string `json:"agent_id"`
ProviderID *string `json:"provider_id"`
ProviderType *string `json:"provider_type"`
ModelID *string `json:"model_id"`
Window string `json:"window"`
MaxTokens *int64 `json:"max_tokens"`
MaxCostMicros *int64 `json:"max_cost_micros"`
MaxCostUSD *float64 `json:"max_cost_usd"`
Enabled *bool `json:"enabled"`
Priority *int `json:"priority"`
}
type overrideBody struct {
ProviderID string `json:"provider_id"`
ProviderType string `json:"provider_type"`
ModelID string `json:"model_id"`
Pricing store.UsagePricingFields `json:"pricing"`
Enabled *bool `json:"enabled"`
}
func (b policyBody) toPolicy(tenantID uuid.UUID) (store.UsageCapPolicy, error) {
p := store.UsageCapPolicy{TenantID: tenantID, Window: b.Window, Enabled: true, Priority: 100}
if b.Enabled != nil {
p.Enabled = *b.Enabled
}
if b.Priority != nil {
p.Priority = *b.Priority
}
if p.Window == "" {
p.Window = store.UsageCapWindowDay
}
var err error
p.AgentID, err = parseOptionalUUIDStrict("agent_id", b.AgentID)
if err != nil {
return p, err
}
p.ProviderID, err = parseOptionalUUIDStrict("provider_id", b.ProviderID)
if err != nil {
return p, err
}
if b.ProviderType != nil {
p.ProviderType = *b.ProviderType
}
if b.ModelID != nil {
p.ModelID = *b.ModelID
}
p.MaxTokens = b.MaxTokens
p.MaxCostMicros = maxCostMicros(b.MaxCostMicros, b.MaxCostUSD)
return p, nil
}
func (b policyBody) toPatch() (store.UsageCapPolicyPatch, error) {
var patch store.UsageCapPolicyPatch
if b.AgentID != nil {
v, err := parseOptionalUUIDStrict("agent_id", b.AgentID)
if err != nil {
return patch, err
}
patch.AgentID = &v
}
if b.ProviderID != nil {
v, err := parseOptionalUUIDStrict("provider_id", b.ProviderID)
if err != nil {
return patch, err
}
patch.ProviderID = &v
}
patch.ProviderType = b.ProviderType
patch.ModelID = b.ModelID
if b.Window != "" {
patch.Window = &b.Window
}
if b.MaxTokens != nil {
patch.MaxTokens = &b.MaxTokens
}
if b.MaxCostMicros != nil || b.MaxCostUSD != nil {
v := maxCostMicros(b.MaxCostMicros, b.MaxCostUSD)
patch.MaxCostMicros = &v
}
patch.Enabled = b.Enabled
patch.Priority = b.Priority
return patch, nil
}
func policyPatchFromBody(bodyBytes []byte) (store.UsageCapPolicyPatch, error) {
var body policyBody
if err := json.Unmarshal(bodyBytes, &body); err != nil {
return store.UsageCapPolicyPatch{}, err
}
patch, err := body.toPatch()
if err != nil {
return patch, err
}
var raw map[string]json.RawMessage
if err := json.Unmarshal(bodyBytes, &raw); err == nil {
if isJSONNull(raw["max_tokens"]) {
var v *int64
patch.MaxTokens = &v
}
if isJSONNull(raw["max_cost_micros"]) || isJSONNull(raw["max_cost_usd"]) {
var v *int64
patch.MaxCostMicros = &v
}
}
return patch, nil
}
func isJSONNull(raw json.RawMessage) bool {
return strings.TrimSpace(string(raw)) == "null"
}
func tenantIDOrMaster(r *http.Request) uuid.UUID {
if tid := store.TenantIDFromContext(r.Context()); tid != uuid.Nil {
return tid
}
return store.MasterTenantID
}
func queryInt(r *http.Request, key string, fallback int) int {
n, err := strconv.Atoi(r.URL.Query().Get(key))
if err != nil || n <= 0 {
return fallback
}
return n
}
func parseOptionalUUIDStrict(field string, raw *string) (*uuid.UUID, error) {
if raw == nil || *raw == "" {
return nil, nil
}
id, err := uuid.Parse(*raw)
if err != nil {
return nil, fmt.Errorf("invalid %s", field)
}
return &id, nil
}
func maxCostMicros(micros *int64, usd *float64) *int64 {
if micros != nil {
return micros
}
if usd == nil {
return nil
}
v := int64(*usd * 1_000_000)
return &v
}
@@ -0,0 +1,35 @@
package http
import "testing"
func TestPolicyPatchFromBodyClearsTokenAndCostLimits(t *testing.T) {
patch, err := policyPatchFromBody([]byte(`{"max_tokens":null,"max_cost_usd":null}`))
if err != nil {
t.Fatalf("policyPatchFromBody: %v", err)
}
if patch.MaxTokens == nil {
t.Fatal("MaxTokens patch missing")
}
if *patch.MaxTokens != nil {
t.Fatalf("MaxTokens = %v, want nil clear", **patch.MaxTokens)
}
if patch.MaxCostMicros == nil {
t.Fatal("MaxCostMicros patch missing")
}
if *patch.MaxCostMicros != nil {
t.Fatalf("MaxCostMicros = %v, want nil clear", **patch.MaxCostMicros)
}
}
func TestPolicyPatchFromBodyPreservesProvidedCostUSD(t *testing.T) {
patch, err := policyPatchFromBody([]byte(`{"max_cost_usd":12.5}`))
if err != nil {
t.Fatalf("policyPatchFromBody: %v", err)
}
if patch.MaxCostMicros == nil || *patch.MaxCostMicros == nil {
t.Fatal("MaxCostMicros patch missing")
}
if got := **patch.MaxCostMicros; got != 12_500_000 {
t.Fatalf("MaxCostMicros = %d, want 12500000", got)
}
}
+17
View File
@@ -0,0 +1,17 @@
package http
import (
"testing"
"github.com/google/uuid"
)
func TestUsageCapPolicyBodyRejectsInvalidUUID(t *testing.T) {
invalid := "not-a-uuid"
if _, err := (policyBody{AgentID: &invalid}).toPolicy(uuid.New()); err == nil {
t.Fatal("toPolicy accepted invalid agent_id")
}
if _, err := (policyBody{ProviderID: &invalid}).toPatch(); err == nil {
t.Fatal("toPatch accepted invalid provider_id")
}
}