mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-15 18:15:47 +00:00
Merge remote-tracking branch 'origin/dev' into codex/issue-48-google-workspace-cli
# Conflicts: # docs/project-changelog.md
This commit is contained in:
commit
3a749ff91c
189 files changed
+8860
-848
No files matched your search
@@ -7,12 +7,14 @@ import (
|
||||
kg "github.com/nextlevelbuilder/goclaw/internal/knowledgegraph"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
|
||||
)
|
||||
|
||||
// KnowledgeGraphHandler handles KG entity/relation management endpoints.
|
||||
type KnowledgeGraphHandler struct {
|
||||
store store.KnowledgeGraphStore
|
||||
providerReg *providers.Registry
|
||||
usageCaps *usagecaps.Service
|
||||
}
|
||||
|
||||
// NewKnowledgeGraphHandler creates a handler for KG management endpoints.
|
||||
@@ -20,6 +22,10 @@ func NewKnowledgeGraphHandler(s store.KnowledgeGraphStore, providerReg *provider
|
||||
return &KnowledgeGraphHandler{store: s, providerReg: providerReg}
|
||||
}
|
||||
|
||||
func (h *KnowledgeGraphHandler) SetUsageCapService(s *usagecaps.Service) {
|
||||
h.usageCaps = s
|
||||
}
|
||||
|
||||
// NewExtractor creates an Extractor from the given provider name and model.
|
||||
func (h *KnowledgeGraphHandler) NewExtractor(ctx context.Context, providerName, model string, minConfidence float64) *kg.Extractor {
|
||||
if h.providerReg == nil || providerName == "" || model == "" {
|
||||
@@ -29,7 +35,9 @@ func (h *KnowledgeGraphHandler) NewExtractor(ctx context.Context, providerName,
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return kg.NewExtractor(p, model, minConfidence)
|
||||
extractor := kg.NewExtractor(p, model, minConfidence)
|
||||
extractor.SetUsageCapService(h.usageCaps)
|
||||
return extractor
|
||||
}
|
||||
|
||||
// RegisterRoutes registers all KG routes on the given mux.
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/i18n"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
@@ -156,6 +157,10 @@ func (h *KnowledgeGraphHandler) handleTraverse(w http.ResponseWriter, r *http.Re
|
||||
func (h *KnowledgeGraphHandler) handleExtract(w http.ResponseWriter, r *http.Request) {
|
||||
locale := extractLocale(r)
|
||||
agentID := r.PathValue("agentID")
|
||||
callCtx := r.Context()
|
||||
if parsedAgentID, err := uuid.Parse(agentID); err == nil {
|
||||
callCtx = store.WithAgentID(callCtx, parsedAgentID)
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Text string `json:"text"`
|
||||
@@ -176,13 +181,13 @@ func (h *KnowledgeGraphHandler) handleExtract(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
extractor := h.NewExtractor(r.Context(), body.Provider, body.Model, body.MinConf)
|
||||
extractor := h.NewExtractor(callCtx, body.Provider, body.Model, body.MinConf)
|
||||
if extractor == nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidProviderOrModel)})
|
||||
return
|
||||
}
|
||||
|
||||
result, err := extractor.Extract(r.Context(), body.Text)
|
||||
result, err := extractor.Extract(callCtx, body.Text)
|
||||
if err != nil {
|
||||
slog.Warn("kg.extract failed", "error", err)
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
||||
@@ -213,10 +218,10 @@ func (h *KnowledgeGraphHandler) handleExtract(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"entities": len(result.Entities),
|
||||
"relations": len(result.Relations),
|
||||
"dedup_merged": dedupMerged,
|
||||
"dedup_flagged": dedupFlagged,
|
||||
"entities": len(result.Entities),
|
||||
"relations": len(result.Relations),
|
||||
"dedup_merged": dedupMerged,
|
||||
"dedup_flagged": dedupFlagged,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providerresolve"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
|
||||
)
|
||||
|
||||
// PendingMessagesHandler handles pending message HTTP endpoints.
|
||||
@@ -22,6 +23,7 @@ type PendingMessagesHandler struct {
|
||||
maxTokens int // max output tokens for LLM summarization (0 = use default)
|
||||
cfgProvider string // config-level provider override (empty = resolve from agent)
|
||||
cfgModel string // config-level model override (empty = resolve from agent)
|
||||
usageCaps *usagecaps.Service
|
||||
}
|
||||
|
||||
func NewPendingMessagesHandler(s store.PendingMessageStore, agentStore store.AgentStore, providerReg *providers.Registry) *PendingMessagesHandler {
|
||||
@@ -40,6 +42,10 @@ func (h *PendingMessagesHandler) SetProviderModel(provider, model string) {
|
||||
h.cfgModel = model
|
||||
}
|
||||
|
||||
func (h *PendingMessagesHandler) SetUsageCapService(s *usagecaps.Service) {
|
||||
h.usageCaps = s
|
||||
}
|
||||
|
||||
func (h *PendingMessagesHandler) RegisterRoutes(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /v1/pending-messages", h.authMiddleware(h.handleListGroups))
|
||||
mux.HandleFunc("GET /v1/pending-messages/messages", h.authMiddleware(h.handleListMessages))
|
||||
@@ -148,7 +154,7 @@ func (h *PendingMessagesHandler) handleCompact(w http.ResponseWriter, r *http.Re
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(store.WithTenantID(context.Background(), tenantID), 180*time.Second)
|
||||
defer cancel()
|
||||
remaining, err := channels.CompactGroup(ctx, h.store, req.ChannelName, req.HistoryKey, provider, model, keepRecent, h.maxTokens)
|
||||
remaining, err := channels.CompactGroup(ctx, h.store, req.ChannelName, req.HistoryKey, provider, model, keepRecent, h.maxTokens, h.usageCaps)
|
||||
if err != nil {
|
||||
slog.Warn("compact.failed", "channel", req.ChannelName, "key", req.HistoryKey, "error", err)
|
||||
} else {
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/i18n"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
|
||||
)
|
||||
|
||||
// HandleVerifyProviderForTest invokes the verify handler directly without auth
|
||||
@@ -126,8 +127,9 @@ func (h *ProvidersHandler) handleVerifyProvider(w http.ResponseWriter, r *http.R
|
||||
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
ctx = store.WithTenantID(ctx, p.TenantID)
|
||||
|
||||
_, err = provider.Chat(ctx, providers.ChatRequest{
|
||||
reqChat := providers.ChatRequest{
|
||||
Messages: []providers.Message{
|
||||
{Role: "user", Content: "hi"},
|
||||
},
|
||||
@@ -136,6 +138,13 @@ func (h *ProvidersHandler) handleVerifyProvider(w http.ResponseWriter, r *http.R
|
||||
// Use a small but safe value — reasoning models need headroom beyond 1 token.
|
||||
"max_tokens": 50,
|
||||
},
|
||||
}
|
||||
_, err = h.usageCaps.Chat(ctx, provider, reqChat, usagecaps.ChatOptions{
|
||||
TenantID: p.TenantID,
|
||||
ProviderName: p.Name,
|
||||
ModelID: req.Model,
|
||||
Purpose: "provider-verify",
|
||||
MaxOutputTokens: 50,
|
||||
})
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"valid": false, "error": friendlyVerifyError(err)})
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/permissions"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
|
||||
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
|
||||
)
|
||||
|
||||
@@ -39,6 +40,7 @@ type ProvidersHandler struct {
|
||||
tracingStore store.TracingStore // optional: for provider-scoped pool activity
|
||||
agents store.AgentCRUDStore // optional: for provider pool activity agent lookup
|
||||
modelReg providers.ModelRegistry // optional: forward-compat model resolver for Anthropic
|
||||
usageCaps *usagecaps.Service
|
||||
}
|
||||
|
||||
// NewProvidersHandler creates a handler for provider management endpoints.
|
||||
@@ -85,6 +87,10 @@ func (h *ProvidersHandler) SetModelRegistry(r providers.ModelRegistry) {
|
||||
h.modelReg = r
|
||||
}
|
||||
|
||||
func (h *ProvidersHandler) SetUsageCapService(s *usagecaps.Service) {
|
||||
h.usageCaps = s
|
||||
}
|
||||
|
||||
// resolveAPIBase returns the provider's api_base, falling back to config/env if empty.
|
||||
// For Ollama/OllamaCloud providers, applies a safety-net normalization: if the stored
|
||||
// value is missing the /v1 suffix (pre-existing record before write-time normalization),
|
||||
|
||||
+43
-99
@@ -7,7 +7,6 @@ import (
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -70,68 +69,13 @@ func (h *SecureCLIHandler) emitCacheInvalidate(key string) {
|
||||
|
||||
// envKeysFromDecryptedJSON returns sorted env variable names from plaintext env JSON (decrypted blob).
|
||||
func envKeysFromDecryptedJSON(env []byte) []string {
|
||||
empty := []string{}
|
||||
if len(env) == 0 {
|
||||
return empty
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(env, &m); err != nil {
|
||||
return empty
|
||||
}
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
return store.SecureCLIEnvKeys(env)
|
||||
}
|
||||
|
||||
// mergeSecureCLIEnv merges incoming env from the UI with existing stored env.
|
||||
// Incoming defines the full set of keys shown in the form: keys omitted were removed.
|
||||
// Empty string means "keep existing value" for that key when it already exists.
|
||||
func mergeSecureCLIEnv(existingJSON []byte, incoming map[string]any) (map[string]string, error) {
|
||||
existing := map[string]string{}
|
||||
if len(existingJSON) > 0 {
|
||||
if err := json.Unmarshal(existingJSON, &existing); err != nil {
|
||||
return nil, fmt.Errorf("parse existing env: %w", err)
|
||||
}
|
||||
}
|
||||
out := make(map[string]string)
|
||||
for k, v := range incoming {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
sv, err := envValueAsString(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid environment variable value")
|
||||
}
|
||||
if sv != "" {
|
||||
out[k] = sv
|
||||
continue
|
||||
}
|
||||
if ev, ok := existing[k]; ok {
|
||||
out[k] = ev
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func envValueAsString(v any) (string, error) {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
return t, nil
|
||||
case float64:
|
||||
return fmt.Sprint(t), nil
|
||||
case bool:
|
||||
if t {
|
||||
return "true", nil
|
||||
}
|
||||
return "false", nil
|
||||
case nil:
|
||||
return "", nil
|
||||
default:
|
||||
return "", fmt.Errorf("value must be a string")
|
||||
}
|
||||
func populateBinaryEnvResponse(b *store.SecureCLIBinary) {
|
||||
b.EnvKeys = store.SecureCLIEnvKeys(b.EncryptedEnv)
|
||||
b.Env = store.SanitizeSecureCLIEnvJSON(b.EncryptedEnv)
|
||||
b.EncryptedEnv = nil
|
||||
}
|
||||
|
||||
func (h *SecureCLIHandler) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -142,27 +86,26 @@ func (h *SecureCLIHandler) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgFailedToList, "CLI credentials")})
|
||||
return
|
||||
}
|
||||
// Never send env values; only variable names for editing.
|
||||
// Sensitive env values stay masked; value-kind entries are returned for editing.
|
||||
for i := range result {
|
||||
result[i].EnvKeys = envKeysFromDecryptedJSON(result[i].EncryptedEnv)
|
||||
result[i].EncryptedEnv = nil
|
||||
populateBinaryEnvResponse(&result[i])
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"items": result})
|
||||
}
|
||||
|
||||
// secureCLICreateRequest supports both preset-based and custom creation.
|
||||
type secureCLICreateRequest struct {
|
||||
Preset string `json:"preset,omitempty"` // auto-fill from preset
|
||||
BinaryName string `json:"binary_name"`
|
||||
BinaryPath *string `json:"binary_path,omitempty"`
|
||||
Description string `json:"description"`
|
||||
Env map[string]string `json:"env"` // plaintext env vars (encrypted by store)
|
||||
DenyArgs json.RawMessage `json:"deny_args,omitempty"`
|
||||
DenyVerbose json.RawMessage `json:"deny_verbose,omitempty"`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
|
||||
Tips string `json:"tips,omitempty"`
|
||||
IsGlobal *bool `json:"is_global,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Preset string `json:"preset,omitempty"` // auto-fill from preset
|
||||
BinaryName string `json:"binary_name"`
|
||||
BinaryPath *string `json:"binary_path,omitempty"`
|
||||
Description string `json:"description"`
|
||||
Env json.RawMessage `json:"env"` // plaintext env vars or env entry objects (encrypted by store)
|
||||
DenyArgs json.RawMessage `json:"deny_args,omitempty"`
|
||||
DenyVerbose json.RawMessage `json:"deny_verbose,omitempty"`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty"`
|
||||
Tips string `json:"tips,omitempty"`
|
||||
IsGlobal *bool `json:"is_global,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
func (h *SecureCLIHandler) handleCreate(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -209,10 +152,14 @@ func (h *SecureCLIHandler) handleCreate(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
// Serialize env as JSON bytes (store layer encrypts)
|
||||
envJSON, err := json.Marshal(req.Env)
|
||||
envEntries, err := store.ParseSecureCLIEnv(req.Env)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid env"})
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
|
||||
return
|
||||
}
|
||||
envJSON, err := store.SerializeSecureCLIEnv(envEntries)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -240,7 +187,7 @@ func (h *SecureCLIHandler) handleCreate(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
tools.ResetCredentialScrubValues() // clear stale scrub values
|
||||
b.EncryptedEnv = nil // don't return credentials
|
||||
populateBinaryEnvResponse(b)
|
||||
emitAudit(h.msgBus, r, "secure_cli.created", "secure_cli", b.ID.String())
|
||||
h.emitCacheInvalidate(b.ID.String())
|
||||
writeJSON(w, http.StatusCreated, b)
|
||||
@@ -260,8 +207,7 @@ func (h *SecureCLIHandler) handleGet(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
b.EnvKeys = envKeysFromDecryptedJSON(b.EncryptedEnv)
|
||||
b.EncryptedEnv = nil // don't expose credential values
|
||||
populateBinaryEnvResponse(b)
|
||||
writeJSON(w, http.StatusOK, b)
|
||||
}
|
||||
|
||||
@@ -293,25 +239,23 @@ func (h *SecureCLIHandler) handleUpdate(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
// If env is updated, merge with stored env so empty values mean "keep existing secret".
|
||||
if envVal, ok := updates["env"]; ok {
|
||||
if envMap, isMap := envVal.(map[string]any); isMap {
|
||||
cur, err := h.store.Get(r.Context(), id)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusNotFound, map[string]string{"error": i18n.T(locale, i18n.MsgNotFound, "credential", id.String())})
|
||||
return
|
||||
}
|
||||
merged, err := mergeSecureCLIEnv(cur.EncryptedEnv, envMap)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
envJSON, err := json.Marshal(merged)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid env"})
|
||||
return
|
||||
}
|
||||
updates["encrypted_env"] = string(envJSON)
|
||||
delete(updates, "env")
|
||||
envJSON, err := json.Marshal(envVal)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
|
||||
return
|
||||
}
|
||||
cur, err := h.store.Get(r.Context(), id)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusNotFound, map[string]string{"error": i18n.T(locale, i18n.MsgNotFound, "credential", id.String())})
|
||||
return
|
||||
}
|
||||
merged, err := store.MergeSecureCLIEnv(cur.EncryptedEnv, envJSON)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
|
||||
return
|
||||
}
|
||||
updates["encrypted_env"] = string(merged)
|
||||
delete(updates, "env")
|
||||
}
|
||||
|
||||
if err := h.store.Update(r.Context(), id, updates); err != nil {
|
||||
|
||||
@@ -68,7 +68,7 @@ func (h *SecureCLIGrantHandler) RegisterRoutes(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /v1/cli-credentials/{id}/agent-grants/{grantId}", auth(h.handleGet))
|
||||
mux.HandleFunc("PUT /v1/cli-credentials/{id}/agent-grants/{grantId}", auth(h.handleUpdate))
|
||||
mux.HandleFunc("DELETE /v1/cli-credentials/{id}/agent-grants/{grantId}", auth(h.handleDelete))
|
||||
// POST (not GET) to prevent caching and satisfy CSRF semantics per Red Team C1.
|
||||
// POST keeps revealed secret material out of URL/history and avoids query caching.
|
||||
mux.HandleFunc("POST /v1/cli-credentials/{id}/agent-grants/{grantId}/env:reveal", auth(h.handleRevealEnv))
|
||||
}
|
||||
|
||||
@@ -76,45 +76,41 @@ func (h *SecureCLIGrantHandler) RegisterRoutes(mux *http.ServeMux) {
|
||||
// EnvVars is optional; plaintext values are encrypted by the store layer.
|
||||
// Clients MUST NOT send encrypted_env — that field is never accepted from the wire.
|
||||
type grantCreateRequest struct {
|
||||
AgentID uuid.UUID `json:"agent_id"`
|
||||
EnvVars map[string]string `json:"env_vars,omitempty"`
|
||||
DenyArgs *json.RawMessage `json:"deny_args,omitempty"`
|
||||
DenyVerbose *json.RawMessage `json:"deny_verbose,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
Tips *string `json:"tips,omitempty"`
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
AgentID uuid.UUID `json:"agent_id"`
|
||||
EnvVars json.RawMessage `json:"env_vars,omitempty"`
|
||||
DenyArgs *json.RawMessage `json:"deny_args,omitempty"`
|
||||
DenyVerbose *json.RawMessage `json:"deny_verbose,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
Tips *string `json:"tips,omitempty"`
|
||||
Enabled *bool `json:"enabled,omitempty"`
|
||||
}
|
||||
|
||||
// populateGrantEnvFields sets EnvKeys (sorted) and EnvSet from the grant's decrypted env bytes.
|
||||
// Plaintext values are never exposed — only key names.
|
||||
// populateGrantEnvFields sets sorted key names, env presence, and sanitized entries.
|
||||
func populateGrantEnvFields(g *store.SecureCLIAgentGrant) {
|
||||
if len(g.EncryptedEnv) == 0 {
|
||||
g.EnvKeys = []string{}
|
||||
g.Env = nil
|
||||
g.EnvSet = false
|
||||
return
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(g.EncryptedEnv, &m); err != nil {
|
||||
g.EnvKeys = []string{}
|
||||
g.EnvSet = false
|
||||
return
|
||||
}
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
keys := store.SecureCLIEnvKeys(g.EncryptedEnv)
|
||||
g.EnvKeys = keys
|
||||
g.Env = store.SanitizeSecureCLIEnvJSON(g.EncryptedEnv)
|
||||
g.EnvSet = len(keys) > 0
|
||||
}
|
||||
|
||||
// validateAndSerializeEnvVars validates env keys/values via denylist and returns serialized JSON.
|
||||
// Returns (nil, 400 error response written) on denial, (jsonBytes, nil) on success.
|
||||
// Never logs env values or keys in error paths.
|
||||
func validateAndSerializeEnvVars(w http.ResponseWriter, locale string, envVars map[string]string) ([]byte, bool) {
|
||||
if len(envVars) == 0 {
|
||||
b, _ := json.Marshal(envVars)
|
||||
return b, true
|
||||
func validateAndSerializeEnvVars(w http.ResponseWriter, locale string, raw json.RawMessage) ([]byte, bool) {
|
||||
envEntries, err := store.ParseSecureCLIEnv(raw)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
|
||||
return nil, false
|
||||
}
|
||||
envVars := make(map[string]string, len(envEntries))
|
||||
for key, entry := range envEntries {
|
||||
envVars[key] = entry.Value
|
||||
}
|
||||
denied, valErr := crypto.ValidateGrantEnvVars(envVars)
|
||||
if valErr != nil {
|
||||
@@ -129,7 +125,7 @@ func validateAndSerializeEnvVars(w http.ResponseWriter, locale string, envVars m
|
||||
})
|
||||
return nil, false
|
||||
}
|
||||
b, err := json.Marshal(envVars)
|
||||
b, err := store.SerializeSecureCLIEnv(envEntries)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, "serialization failed")})
|
||||
return nil, false
|
||||
@@ -249,7 +245,7 @@ func (h *SecureCLIGrantHandler) handleCreate(w http.ResponseWriter, r *http.Requ
|
||||
envJSON, ok := validateAndSerializeEnvVars(w, locale, req.EnvVars)
|
||||
if !ok {
|
||||
// Grant was created but env validation failed; clean it up to avoid orphan row.
|
||||
// Finding #13: log rollback-delete failures for ops visibility.
|
||||
// Log rollback-delete failures so operators can clean up orphan rows.
|
||||
if delErr := h.grants.Delete(r.Context(), g.ID); delErr != nil {
|
||||
slog.Error("secure_cli_grants.create.rollback_delete",
|
||||
"grant_id", g.ID,
|
||||
@@ -261,7 +257,7 @@ func (h *SecureCLIGrantHandler) handleCreate(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
if err := h.grants.UpdateGrantEnv(r.Context(), g.ID, envJSON); err != nil {
|
||||
slog.Error("secure_cli_grants.create.set_env", "grant_id", g.ID, "error", err)
|
||||
// Finding #13: log rollback-delete failures for ops visibility.
|
||||
// Log rollback-delete failures so operators can clean up orphan rows.
|
||||
if delErr := h.grants.Delete(r.Context(), g.ID); delErr != nil {
|
||||
slog.Error("secure_cli_grants.create.rollback_delete",
|
||||
"grant_id", g.ID,
|
||||
@@ -323,7 +319,7 @@ func (h *SecureCLIGrantHandler) handleUpdate(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
if allowedScalar[k] {
|
||||
var decoded any
|
||||
// Finding #3: return 400 on Unmarshal failure — silent discard means admin
|
||||
// Return 400 on unmarshal failure; silent discard means admin
|
||||
// thinks they applied a change (e.g. enabled: "false") but the grant is unchanged.
|
||||
if err := json.Unmarshal(v, &decoded); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{
|
||||
@@ -335,7 +331,7 @@ func (h *SecureCLIGrantHandler) handleUpdate(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
}
|
||||
// 3-state env_vars semantics: absent=skip, null=clear, {...}=replace.
|
||||
// Finding #15: {} (empty map) is treated as clear — same as null.
|
||||
// Empty map is treated as clear, same as null.
|
||||
// TS type: absent | null | Record<string,string> — see ui/web/src/types/cli-credential.ts.
|
||||
var envJSON []byte
|
||||
envPresent := false
|
||||
@@ -343,18 +339,22 @@ func (h *SecureCLIGrantHandler) handleUpdate(w http.ResponseWriter, r *http.Requ
|
||||
envPresent = true
|
||||
var envPtr *map[string]string
|
||||
if string(envRaw) != "null" {
|
||||
var m map[string]string
|
||||
if err := json.Unmarshal(envRaw, &m); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, "env_vars must be a string map")})
|
||||
envEntries, err := store.ParseSecureCLIEnv(envRaw)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, "env_vars must be a string map or env entries")})
|
||||
return
|
||||
}
|
||||
m := make(map[string]string, len(envEntries))
|
||||
for key, entry := range envEntries {
|
||||
m[key] = entry.Value
|
||||
}
|
||||
envPtr = &m
|
||||
}
|
||||
// envPtr == nil → clear; envPtr != nil → replace.
|
||||
// Note: envPtr pointing to an empty map ({}) is treated as clear (same as null) —
|
||||
// envJSON stays nil and UpdateGrantEnv(nil) removes the override.
|
||||
if envPtr != nil && len(*envPtr) > 0 {
|
||||
j, ok := validateAndSerializeEnvVars(w, locale, *envPtr)
|
||||
j, ok := validateAndSerializeEnvVars(w, locale, envRaw)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
@@ -430,8 +430,8 @@ func (h *SecureCLIGrantHandler) handleRevealEnv(w http.ResponseWriter, r *http.R
|
||||
locale := store.LocaleFromContext(ctx)
|
||||
|
||||
// Rate limit: 10 reveals/min per authenticated caller (context UserID).
|
||||
// Finding #2: require non-empty UserID from authenticated context.
|
||||
// If UserID is empty, the auth middleware failed to populate it — reject rather
|
||||
// Require non-empty UserID from authenticated context.
|
||||
// If UserID is empty, auth middleware failed to populate it — reject rather
|
||||
// than fall back to a spoofable header or IP address.
|
||||
callerID := store.UserIDFromContext(ctx)
|
||||
if callerID == "" {
|
||||
@@ -476,8 +476,8 @@ func (h *SecureCLIGrantHandler) handleRevealEnv(w http.ResponseWriter, r *http.R
|
||||
writeJSON(w, http.StatusOK, map[string]any{"env_vars": map[string]string{}})
|
||||
return
|
||||
}
|
||||
var envVars map[string]string
|
||||
if err := json.Unmarshal(g.EncryptedEnv, &envVars); err != nil {
|
||||
envVars, err := store.FlattenSecureCLIEnv(g.EncryptedEnv)
|
||||
if err != nil {
|
||||
slog.Error("secure_cli_grants.reveal.parse", "grant_id", g.ID, "error", err)
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "parse grant env")})
|
||||
return
|
||||
|
||||
@@ -3,6 +3,7 @@ package http
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -59,8 +60,16 @@ func (s *fakeSecureCLIGrantStore) Delete(context.Context, uuid.UUID) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakeSecureCLIGrantStore) ListByBinary(context.Context, uuid.UUID) ([]store.SecureCLIAgentGrant, error) {
|
||||
return nil, nil
|
||||
func (s *fakeSecureCLIGrantStore) ListByBinary(_ context.Context, binaryID uuid.UUID) ([]store.SecureCLIAgentGrant, error) {
|
||||
grants := make([]store.SecureCLIAgentGrant, 0, len(s.grants))
|
||||
for _, grant := range s.grants {
|
||||
if grant == nil || grant.BinaryID != binaryID {
|
||||
continue
|
||||
}
|
||||
cp := *grant
|
||||
grants = append(grants, cp)
|
||||
}
|
||||
return grants, nil
|
||||
}
|
||||
|
||||
func (s *fakeSecureCLIGrantStore) ListByAgent(context.Context, uuid.UUID) ([]store.SecureCLIAgentGrant, error) {
|
||||
@@ -202,3 +211,42 @@ func TestSecureCLIGrantUpdateRejectsInvalidEnvVarsBeforeScalarUpdate(t *testing.
|
||||
t.Fatal("invalid env_vars request must not persist scalar grant updates")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecureCLIGrantGetSanitizesMixedEnv(t *testing.T) {
|
||||
binaryID := uuid.New()
|
||||
grantID := uuid.New()
|
||||
fake := &fakeSecureCLIGrantStore{
|
||||
grants: map[uuid.UUID]*store.SecureCLIAgentGrant{
|
||||
grantID: {
|
||||
BaseModel: store.BaseModel{ID: grantID},
|
||||
BinaryID: binaryID,
|
||||
AgentID: uuid.New(),
|
||||
Enabled: true,
|
||||
EncryptedEnv: []byte(`{"TOKEN":"secret-token","PUBLIC_BASE_URL":{"kind":"value","value":"https://goclaw.sh"}}`),
|
||||
},
|
||||
},
|
||||
}
|
||||
h := NewSecureCLIGrantHandler(fake, nil, nil)
|
||||
rr, req := requestWithGrantPath(http.MethodGet, nil, binaryID, grantID)
|
||||
|
||||
h.handleGet(rr, req)
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if strings.Contains(rr.Body.String(), "secret-token") {
|
||||
t.Fatalf("sensitive grant env leaked in response: %s", rr.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Env map[string]store.SecureCLIEnvResponseEntry `json:"env"`
|
||||
}
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.Env["TOKEN"].Masked || got.Env["TOKEN"].Value != nil {
|
||||
t.Fatalf("TOKEN not masked: %#v", got.Env["TOKEN"])
|
||||
}
|
||||
if got.Env["PUBLIC_BASE_URL"].Value == nil || *got.Env["PUBLIC_BASE_URL"].Value != "https://goclaw.sh" {
|
||||
t.Fatalf("PUBLIC_BASE_URL not returned: %#v", got.Env["PUBLIC_BASE_URL"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
type fakeSecureCLIStore struct {
|
||||
binary *store.SecureCLIBinary
|
||||
user *store.SecureCLIUserCredential
|
||||
}
|
||||
|
||||
func (s *fakeSecureCLIStore) Create(context.Context, *store.SecureCLIBinary) error { return nil }
|
||||
func (s *fakeSecureCLIStore) Get(context.Context, uuid.UUID) (*store.SecureCLIBinary, error) {
|
||||
cp := *s.binary
|
||||
return &cp, nil
|
||||
}
|
||||
func (s *fakeSecureCLIStore) Update(context.Context, uuid.UUID, map[string]any) error { return nil }
|
||||
func (s *fakeSecureCLIStore) Delete(context.Context, uuid.UUID) error { return nil }
|
||||
func (s *fakeSecureCLIStore) List(context.Context) ([]store.SecureCLIBinary, error) {
|
||||
cp := *s.binary
|
||||
return []store.SecureCLIBinary{cp}, nil
|
||||
}
|
||||
func (s *fakeSecureCLIStore) LookupByBinary(context.Context, string, *uuid.UUID, string) (*store.SecureCLIBinary, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *fakeSecureCLIStore) ListEnabled(context.Context) ([]store.SecureCLIBinary, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *fakeSecureCLIStore) ListForAgent(context.Context, uuid.UUID) ([]store.SecureCLIBinary, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (s *fakeSecureCLIStore) IsRegisteredBinary(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
func (s *fakeSecureCLIStore) GetUserCredentials(context.Context, uuid.UUID, string) (*store.SecureCLIUserCredential, error) {
|
||||
cp := *s.user
|
||||
return &cp, nil
|
||||
}
|
||||
func (s *fakeSecureCLIStore) SetUserCredentials(context.Context, uuid.UUID, string, []byte) error {
|
||||
return nil
|
||||
}
|
||||
func (s *fakeSecureCLIStore) DeleteUserCredentials(context.Context, uuid.UUID, string) error {
|
||||
return nil
|
||||
}
|
||||
func (s *fakeSecureCLIStore) ListUserCredentials(context.Context, uuid.UUID) ([]store.SecureCLIUserCredential, error) {
|
||||
cp := *s.user
|
||||
return []store.SecureCLIUserCredential{cp}, nil
|
||||
}
|
||||
|
||||
func TestSecureCLIGetSanitizesMixedEnv(t *testing.T) {
|
||||
id := uuid.New()
|
||||
h := NewSecureCLIHandler(&fakeSecureCLIStore{
|
||||
binary: &store.SecureCLIBinary{
|
||||
BaseModel: store.BaseModel{ID: id},
|
||||
BinaryName: "gh",
|
||||
EncryptedEnv: []byte(`{"TOKEN":"secret-token","PUBLIC_BASE_URL":{"kind":"value","value":"https://goclaw.sh"}}`),
|
||||
},
|
||||
}, nil)
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/cli-credentials/"+id.String(), nil)
|
||||
req.SetPathValue("id", id.String())
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
h.handleGet(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "secret-token") {
|
||||
t.Fatalf("sensitive env leaked in response: %s", rec.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Env map[string]store.SecureCLIEnvResponseEntry `json:"env"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.Env["TOKEN"].Masked || got.Env["TOKEN"].Value != nil {
|
||||
t.Fatalf("TOKEN not masked: %#v", got.Env["TOKEN"])
|
||||
}
|
||||
if got.Env["PUBLIC_BASE_URL"].Value == nil || *got.Env["PUBLIC_BASE_URL"].Value != "https://goclaw.sh" {
|
||||
t.Fatalf("value env not returned: %#v", got.Env["PUBLIC_BASE_URL"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecureCLIUserCredentialsGetDoesNotReturnLegacySensitiveRaw(t *testing.T) {
|
||||
binaryID := uuid.New()
|
||||
h := NewSecureCLIHandler(&fakeSecureCLIStore{
|
||||
user: &store.SecureCLIUserCredential{
|
||||
ID: uuid.New(),
|
||||
BinaryID: binaryID,
|
||||
UserID: "user-1",
|
||||
EncryptedEnv: []byte(`{"TOKEN":"secret-token","REGION":{"kind":"value","value":"asia-southeast1"}}`),
|
||||
},
|
||||
}, nil)
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/cli-credentials/"+binaryID.String()+"/user-credentials/user-1", nil)
|
||||
req.SetPathValue("id", binaryID.String())
|
||||
req.SetPathValue("userId", "user-1")
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
h.handleGetUserCredentials(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "secret-token") {
|
||||
t.Fatalf("legacy sensitive env leaked in response: %s", rec.Body.String())
|
||||
}
|
||||
var got struct {
|
||||
Env map[string]store.SecureCLIEnvResponseEntry `json:"env"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !got.Env["TOKEN"].Masked || got.Env["TOKEN"].Value != nil {
|
||||
t.Fatalf("TOKEN not masked: %#v", got.Env["TOKEN"])
|
||||
}
|
||||
if got.Env["REGION"].Value == nil || *got.Env["REGION"].Value != "asia-southeast1" {
|
||||
t.Fatalf("REGION not returned: %#v", got.Env["REGION"])
|
||||
}
|
||||
}
|
||||
@@ -25,13 +25,14 @@ func (h *SecureCLIHandler) handleListUserCredentials(w http.ResponseWriter, r *h
|
||||
}
|
||||
// Return without env values for listing (names only + timestamps)
|
||||
type entry struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
BinaryID uuid.UUID `json:"binary_id"`
|
||||
UserID string `json:"user_id"`
|
||||
HasEnv bool `json:"has_env"`
|
||||
EnvKeys []string `json:"env_keys,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
ID uuid.UUID `json:"id"`
|
||||
BinaryID uuid.UUID `json:"binary_id"`
|
||||
UserID string `json:"user_id"`
|
||||
HasEnv bool `json:"has_env"`
|
||||
EnvKeys []string `json:"env_keys,omitempty"`
|
||||
Env map[string]store.SecureCLIEnvResponseEntry `json:"env,omitempty"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
}
|
||||
entries := make([]entry, 0, len(creds))
|
||||
for _, c := range creds {
|
||||
@@ -42,6 +43,7 @@ func (h *SecureCLIHandler) handleListUserCredentials(w http.ResponseWriter, r *h
|
||||
UserID: c.UserID,
|
||||
HasEnv: len(c.EncryptedEnv) > 0,
|
||||
EnvKeys: envKeys,
|
||||
Env: store.SanitizeSecureCLIEnvJSON(c.EncryptedEnv),
|
||||
CreatedAt: c.CreatedAt,
|
||||
UpdatedAt: c.UpdatedAt,
|
||||
})
|
||||
@@ -73,14 +75,9 @@ func (h *SecureCLIHandler) handleGetUserCredentials(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
// Return decrypted env as JSON object (admin-only endpoint)
|
||||
var envObj any
|
||||
if len(cred.EncryptedEnv) > 0 {
|
||||
_ = json.Unmarshal(cred.EncryptedEnv, &envObj)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"user_id": cred.UserID,
|
||||
"env": envObj,
|
||||
"env": store.SanitizeSecureCLIEnvJSON(cred.EncryptedEnv),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -108,14 +105,23 @@ func (h *SecureCLIHandler) handleSetUserCredentials(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
// Validate env is a JSON object
|
||||
var envCheck map[string]string
|
||||
if err := json.Unmarshal(body.Env, &envCheck); err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": "env must be a JSON object with string values"})
|
||||
existing, err := h.store.GetUserCredentials(r.Context(), binaryID, userID)
|
||||
if err != nil {
|
||||
locale := store.LocaleFromContext(r.Context())
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, err.Error())})
|
||||
return
|
||||
}
|
||||
var existingEnv []byte
|
||||
if existing != nil {
|
||||
existingEnv = existing.EncryptedEnv
|
||||
}
|
||||
envJSON, err := store.MergeSecureCLIEnv(existingEnv, body.Env)
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgGrantEnvValueInvalid, err.Error())})
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.store.SetUserCredentials(r.Context(), binaryID, userID, body.Env); err != nil {
|
||||
if err := h.store.SetUserCredentials(r.Context(), binaryID, userID, envJSON); err != nil {
|
||||
locale := store.LocaleFromContext(r.Context())
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, err.Error())})
|
||||
return
|
||||
|
||||
@@ -21,8 +21,6 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
|
||||
)
|
||||
|
||||
const maxSkillUploadSize = 20 << 20 // 20 MB
|
||||
|
||||
var (
|
||||
aggregateInstallDeps = skills.AggregateMissingDeps
|
||||
installManagedDeps = skills.InstallDeps
|
||||
@@ -40,11 +38,13 @@ type SkillsHandler struct {
|
||||
tenantStore store.TenantStore
|
||||
db *sql.DB // for export/import direct queries
|
||||
uploadLocks sync.Map // per-slug mutex; bounded by validated slug set, entries are tiny (*sync.Mutex)
|
||||
uploadLimitCfg config.SkillsConfig
|
||||
systemConfigs store.SystemConfigStore
|
||||
}
|
||||
|
||||
// NewSkillsHandler creates a handler for skill management endpoints.
|
||||
func NewSkillsHandler(skills store.SkillManageStore, baseDir, dataDir, bundledDir string, msgBus *bus.MessageBus, tenantCfgStore store.SkillTenantConfigStore, tenantStore store.TenantStore) *SkillsHandler {
|
||||
return &SkillsHandler{skills: skills, baseDir: baseDir, dataDir: dataDir, bundledDir: bundledDir, msgBus: msgBus, tenantCfgStore: tenantCfgStore, tenantStore: tenantStore}
|
||||
return &SkillsHandler{skills: skills, baseDir: baseDir, dataDir: dataDir, bundledDir: bundledDir, msgBus: msgBus, tenantCfgStore: tenantCfgStore, tenantStore: tenantStore, uploadLimitCfg: config.SkillsConfig{MaxUploadSizeMB: config.DefaultSkillMaxUploadSizeMB}}
|
||||
}
|
||||
|
||||
// tenantSkillsDir returns the skills-store directory scoped to the requesting tenant.
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
@@ -225,16 +223,3 @@ func (h *SkillsHandler) handleRevokeUser(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
// --- Helpers ---
|
||||
|
||||
func readZipFile(f *zip.File) (string, error) {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rc.Close()
|
||||
data, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
@@ -43,7 +44,8 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxSkillUploadSize)
|
||||
preParseLimitMB := h.resolvePreParseUploadLimitMB(r.Context())
|
||||
r.Body = http.MaxBytesReader(w, r.Body, skillUploadBodyBytes(preParseLimitMB))
|
||||
|
||||
file, header, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
@@ -70,6 +72,11 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
|
||||
size, err := io.Copy(tmp, file)
|
||||
if err != nil {
|
||||
tmp.Close()
|
||||
var maxBytesErr *http.MaxBytesError
|
||||
if errors.As(err, &maxBytesErr) {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, skillUploadTooLargeMessage(maxBytesErr.Limit, preParseLimitMB))})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "failed to save upload")})
|
||||
return
|
||||
}
|
||||
@@ -141,6 +148,11 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
name, description, slug, frontmatter := skills.ParseSkillFrontmatter(skillContent)
|
||||
uploadLimitMB := h.resolveSkillUploadLimitMB(r.Context(), frontmatter)
|
||||
if size > skillUploadLimitBytes(uploadLimitMB) {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, skillUploadTooLargeMessage(size, uploadLimitMB))})
|
||||
return
|
||||
}
|
||||
if name == "" {
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgRequired, "name in SKILL.md frontmatter")})
|
||||
return
|
||||
@@ -239,17 +251,11 @@ func (h *SkillsHandler) handleUpload(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "failed to create skill file directory")})
|
||||
return
|
||||
}
|
||||
data, err := readZipFile(f)
|
||||
if err != nil {
|
||||
if err := copyZipFileToPath(f, destPath); err != nil {
|
||||
os.RemoveAll(destDir)
|
||||
writeJSON(w, http.StatusBadRequest, map[string]string{"error": i18n.T(locale, i18n.MsgInvalidRequest, "failed to read ZIP entry")})
|
||||
return
|
||||
}
|
||||
if err := os.WriteFile(destPath, []byte(data), 0644); err != nil {
|
||||
os.RemoveAll(destDir)
|
||||
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": i18n.T(locale, i18n.MsgInternalError, "failed to write skill files")})
|
||||
return
|
||||
}
|
||||
if cleanName == "SKILL.md" {
|
||||
wroteSkillMD = true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
skillUploadMaxSizeConfigKey = config.SkillMaxUploadSizeSystemConfigKey
|
||||
skillUploadMultipartOverheadBytes = int64(1 << 20)
|
||||
)
|
||||
|
||||
func (h *SkillsHandler) SetUploadLimitConfig(cfg config.SkillsConfig) {
|
||||
h.uploadLimitCfg = cfg
|
||||
}
|
||||
|
||||
func (h *SkillsHandler) SetSystemConfigStore(s store.SystemConfigStore) {
|
||||
h.systemConfigs = s
|
||||
}
|
||||
|
||||
func (h *SkillsHandler) resolveSkillUploadLimitMB(ctx context.Context, frontmatter map[string]string) int {
|
||||
if mb, ok := h.resolveTenantSkillUploadLimitMB(ctx); ok {
|
||||
return mb
|
||||
}
|
||||
if mb, ok := parseSkillUploadLimitMB(frontmatter["max_upload_size_mb"]); ok {
|
||||
return config.ClampSkillMaxUploadSizeMB(mb)
|
||||
}
|
||||
return h.uploadLimitCfg.EffectiveMaxUploadSizeMB()
|
||||
}
|
||||
|
||||
func (h *SkillsHandler) resolvePreParseUploadLimitMB(ctx context.Context) int {
|
||||
if mb, ok := h.resolveTenantSkillUploadLimitMB(ctx); ok {
|
||||
return mb
|
||||
}
|
||||
return config.MaxSkillMaxUploadSizeMB
|
||||
}
|
||||
|
||||
func (h *SkillsHandler) resolveTenantSkillUploadLimitMB(ctx context.Context) (int, bool) {
|
||||
if h.systemConfigs == nil {
|
||||
return 0, false
|
||||
}
|
||||
raw, err := h.systemConfigs.Get(ctx, skillUploadMaxSizeConfigKey)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
mb, ok := parseSkillUploadLimitMB(raw)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return config.ClampSkillMaxUploadSizeMB(mb), true
|
||||
}
|
||||
|
||||
func parseSkillUploadLimitMB(raw string) (int, bool) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return 0, false
|
||||
}
|
||||
mb, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return mb, true
|
||||
}
|
||||
|
||||
func skillUploadLimitBytes(limitMB int) int64 {
|
||||
return int64(config.ClampSkillMaxUploadSizeMB(limitMB)) << 20
|
||||
}
|
||||
|
||||
func skillUploadBodyBytes(limitMB int) int64 {
|
||||
return skillUploadLimitBytes(limitMB) + skillUploadMultipartOverheadBytes
|
||||
}
|
||||
|
||||
func skillUploadTooLargeMessage(size int64, limitMB int) string {
|
||||
return fmt.Sprintf("skill ZIP size %s exceeds %d MB limit", formatUploadBytes(size), limitMB)
|
||||
}
|
||||
|
||||
func formatUploadBytes(size int64) string {
|
||||
const mb = int64(1 << 20)
|
||||
if size >= mb {
|
||||
return fmt.Sprintf("%.1f MB", float64(size)/float64(mb))
|
||||
}
|
||||
return fmt.Sprintf("%d bytes", size)
|
||||
}
|
||||
@@ -12,11 +12,13 @@ import (
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/config"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/skills"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
|
||||
@@ -670,6 +672,65 @@ func TestHandleUpload_ReturnsGrantErrors(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveSkillUploadLimitMBPrecedenceAndClamp(t *testing.T) {
|
||||
ctx := store.WithTenantID(context.Background(), store.MasterTenantID)
|
||||
handler, _, _, _ := newTestUploadHandler(t)
|
||||
handler.SetUploadLimitConfig(config.SkillsConfig{MaxUploadSizeMB: 30})
|
||||
|
||||
if got := handler.resolveSkillUploadLimitMB(ctx, nil); got != 30 {
|
||||
t.Fatalf("global limit = %d, want 30", got)
|
||||
}
|
||||
if got := handler.resolveSkillUploadLimitMB(ctx, map[string]string{"max_upload_size_mb": "100"}); got != 100 {
|
||||
t.Fatalf("frontmatter limit = %d, want 100", got)
|
||||
}
|
||||
|
||||
handler.SetSystemConfigStore(&skillUploadSystemConfigStore{data: map[string]string{skillUploadMaxSizeConfigKey: "40"}})
|
||||
if got := handler.resolveSkillUploadLimitMB(ctx, map[string]string{"max_upload_size_mb": "100"}); got != 40 {
|
||||
t.Fatalf("tenant limit = %d, want 40", got)
|
||||
}
|
||||
|
||||
handler.SetSystemConfigStore(&skillUploadSystemConfigStore{data: map[string]string{skillUploadMaxSizeConfigKey: "999"}})
|
||||
if got := handler.resolveSkillUploadLimitMB(ctx, nil); got != config.MaxSkillMaxUploadSizeMB {
|
||||
t.Fatalf("clamped tenant limit = %d, want %d", got, config.MaxSkillMaxUploadSizeMB)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadRejectsZipAboveConfiguredLimit(t *testing.T) {
|
||||
handler, _, ctx, _ := newTestUploadHandler(t)
|
||||
handler.SetUploadLimitConfig(config.SkillsConfig{MaxUploadSizeMB: 1})
|
||||
|
||||
req := newZipUploadRequestWithBinary(t, ctx, map[string][]byte{
|
||||
"SKILL.md": []byte(skillMarkdown("Large Skill", "large-skill")),
|
||||
"large.bin": bytes.Repeat([]byte("x"), (1<<20)+1),
|
||||
})
|
||||
w := httptest.NewRecorder()
|
||||
handler.handleUpload(w, req)
|
||||
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), "exceeds") || !strings.Contains(w.Body.String(), "1 MB") {
|
||||
t.Fatalf("body = %s, want configured upload limit error", w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadAllowsFrontmatterLimitAboveGlobalDefault(t *testing.T) {
|
||||
handler, _, ctx, _ := newTestUploadHandler(t)
|
||||
handler.SetUploadLimitConfig(config.SkillsConfig{MaxUploadSizeMB: 1})
|
||||
skillMD := "---\nname: Video Skill\nslug: video-skill\nmax_upload_size_mb: 2\n---\nSkill body\n"
|
||||
|
||||
req := newZipUploadRequestWithBinary(t, ctx, map[string][]byte{
|
||||
"SKILL.md": []byte(skillMD),
|
||||
"large.bin": bytes.Repeat([]byte("x"), (1<<20)+(128<<10)),
|
||||
})
|
||||
w := httptest.NewRecorder()
|
||||
handler.handleUpload(w, req)
|
||||
|
||||
if w.Code != http.StatusCreated {
|
||||
t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func newTestUploadHandler(t *testing.T) (*SkillsHandler, *skillManageStoreStub, context.Context, string) {
|
||||
t.Helper()
|
||||
|
||||
@@ -738,6 +799,43 @@ func newZipUploadRequestWithManagers(t *testing.T, ctx context.Context, files ma
|
||||
return req.WithContext(ctx)
|
||||
}
|
||||
|
||||
func newZipUploadRequestWithBinary(t *testing.T, ctx context.Context, files map[string][]byte) *http.Request {
|
||||
t.Helper()
|
||||
|
||||
var zipBuf bytes.Buffer
|
||||
zw := zip.NewWriter(&zipBuf)
|
||||
for name, content := range files {
|
||||
header := &zip.FileHeader{Name: name, Method: zip.Store}
|
||||
w, err := zw.CreateHeader(header)
|
||||
if err != nil {
|
||||
t.Fatalf("zip create %s: %v", name, err)
|
||||
}
|
||||
if _, err := w.Write(content); err != nil {
|
||||
t.Fatalf("zip write %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatalf("zip close: %v", err)
|
||||
}
|
||||
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
part, err := mw.CreateFormFile("file", "skill.zip")
|
||||
if err != nil {
|
||||
t.Fatalf("multipart file: %v", err)
|
||||
}
|
||||
if _, err := part.Write(zipBuf.Bytes()); err != nil {
|
||||
t.Fatalf("multipart write: %v", err)
|
||||
}
|
||||
if err := mw.Close(); err != nil {
|
||||
t.Fatalf("multipart close: %v", err)
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/skills/upload", &body)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
return req.WithContext(ctx)
|
||||
}
|
||||
|
||||
func newUploadManagerIDsFormRequest(t *testing.T, raw string) *http.Request {
|
||||
t.Helper()
|
||||
|
||||
@@ -770,6 +868,34 @@ type skillManageStoreStub struct {
|
||||
lastUpdates map[uuid.UUID]map[string]any
|
||||
}
|
||||
|
||||
type skillUploadSystemConfigStore struct {
|
||||
data map[string]string
|
||||
}
|
||||
|
||||
func (s *skillUploadSystemConfigStore) Get(_ context.Context, key string) (string, error) {
|
||||
if v, ok := s.data[key]; ok {
|
||||
return v, nil
|
||||
}
|
||||
return "", errors.New("not found")
|
||||
}
|
||||
|
||||
func (s *skillUploadSystemConfigStore) Set(_ context.Context, key, value string) error {
|
||||
if s.data == nil {
|
||||
s.data = map[string]string{}
|
||||
}
|
||||
s.data[key] = value
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *skillUploadSystemConfigStore) Delete(_ context.Context, key string) error {
|
||||
delete(s.data, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *skillUploadSystemConfigStore) List(_ context.Context) (map[string]string, error) {
|
||||
return s.data, nil
|
||||
}
|
||||
|
||||
type skillGrantCall struct {
|
||||
SkillID uuid.UUID
|
||||
AgentID uuid.UUID
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
const maxSkillMarkdownBytes = 100 << 10
|
||||
|
||||
var errSkillMarkdownTooLarge = errors.New("SKILL.md exceeds 100 KB limit")
|
||||
|
||||
func readZipFile(f *zip.File) (string, error) {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
data, err := io.ReadAll(io.LimitReader(rc, maxSkillMarkdownBytes+1))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(data) > maxSkillMarkdownBytes {
|
||||
return "", errSkillMarkdownTooLarge
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func copyZipFileToPath(f *zip.File, destPath string) error {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
dst, err := os.OpenFile(destPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
_, err = io.Copy(dst, rc)
|
||||
return err
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
|
||||
)
|
||||
|
||||
// Summoning event type constants.
|
||||
@@ -48,14 +49,16 @@ type AgentSummoner struct {
|
||||
agents store.AgentStore
|
||||
providerReg *providers.Registry
|
||||
msgBus *bus.MessageBus
|
||||
usageCaps *usagecaps.Service
|
||||
}
|
||||
|
||||
// NewAgentSummoner creates a summoner backed by the given stores and provider registry.
|
||||
func NewAgentSummoner(agents store.AgentStore, providerReg *providers.Registry, msgBus *bus.MessageBus) *AgentSummoner {
|
||||
func NewAgentSummoner(agents store.AgentStore, providerReg *providers.Registry, msgBus *bus.MessageBus, usageCaps *usagecaps.Service) *AgentSummoner {
|
||||
return &AgentSummoner{
|
||||
agents: agents,
|
||||
providerReg: providerReg,
|
||||
msgBus: msgBus,
|
||||
usageCaps: usageCaps,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,6 +76,7 @@ const singleCallTimeout = 300 * time.Second
|
||||
func (s *AgentSummoner) SummonAgent(agentID uuid.UUID, tenantID uuid.UUID, providerName, model, description string) {
|
||||
ctx, cancel := context.WithTimeout(store.WithTenantID(context.Background(), tenantID), 600*time.Second)
|
||||
defer cancel()
|
||||
ctx = store.WithAgentID(ctx, agentID)
|
||||
|
||||
s.ensureBackfillFiles(ctx, agentID)
|
||||
s.emitEvent(agentID, tenantID, SummonEventStarted, "", "")
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/nextlevelbuilder/goclaw/internal/bus"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
usagecaps "github.com/nextlevelbuilder/goclaw/internal/usage/caps"
|
||||
"github.com/nextlevelbuilder/goclaw/pkg/protocol"
|
||||
)
|
||||
|
||||
@@ -23,6 +24,7 @@ import (
|
||||
func (s *AgentSummoner) RegenerateAgent(agentID uuid.UUID, tenantID uuid.UUID, providerName, model, editPrompt string) {
|
||||
ctx, cancel := context.WithTimeout(store.WithTenantID(context.Background(), tenantID), 300*time.Second)
|
||||
defer cancel()
|
||||
ctx = store.WithAgentID(ctx, agentID)
|
||||
|
||||
s.ensureBackfillFiles(ctx, agentID)
|
||||
|
||||
@@ -121,7 +123,7 @@ func (s *AgentSummoner) generateFiles(ctx context.Context, providerName, model,
|
||||
|
||||
slog.Info("summoning: calling LLM", "provider", providerName, "model", model, "prompt_len", len(prompt))
|
||||
|
||||
resp, err := provider.Chat(ctx, providers.ChatRequest{
|
||||
req := providers.ChatRequest{
|
||||
Messages: []providers.Message{
|
||||
{Role: "system", Content: "You are a file generator. Output ONLY the requested XML-tagged files. No extra commentary."},
|
||||
{Role: "user", Content: prompt},
|
||||
@@ -133,6 +135,12 @@ func (s *AgentSummoner) generateFiles(ctx context.Context, providerName, model,
|
||||
providers.OptSessionKey: summonSessionKey,
|
||||
providers.OptDisableTools: true,
|
||||
},
|
||||
}
|
||||
resp, err := s.usageCaps.Chat(ctx, provider, req, usagecaps.ChatOptions{
|
||||
ProviderName: providerName,
|
||||
ModelID: model,
|
||||
Purpose: "agent-summoner",
|
||||
MaxOutputTokens: 8192,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", providerName, err)
|
||||
|
||||
@@ -0,0 +1,397 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/i18n"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/permissions"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/usage/pricing"
|
||||
)
|
||||
|
||||
type UsageCapsHandler struct {
|
||||
store store.UsageCapStore
|
||||
tenants store.TenantStore
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewUsageCapsHandler(s store.UsageCapStore, tenants store.TenantStore) *UsageCapsHandler {
|
||||
return &UsageCapsHandler{store: s, tenants: tenants, client: &http.Client{Timeout: 30 * time.Second}}
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) RegisterRoutes(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /v1/usage-caps/policies", h.auth(h.handleListPolicies))
|
||||
mux.HandleFunc("POST /v1/usage-caps/policies", h.writeAuth(h.handleCreatePolicy))
|
||||
mux.HandleFunc("PATCH /v1/usage-caps/policies/{id}", h.writeAuth(h.handleUpdatePolicy))
|
||||
mux.HandleFunc("DELETE /v1/usage-caps/policies/{id}", h.writeAuth(h.handleDeletePolicy))
|
||||
mux.HandleFunc("GET /v1/usage-caps/utilization", h.auth(h.handleUtilization))
|
||||
mux.HandleFunc("GET /v1/usage-caps/events", h.auth(h.handleEvents))
|
||||
mux.HandleFunc("POST /v1/model-pricing/sync-openrouter", h.masterAuth(h.handleSyncOpenRouter))
|
||||
mux.HandleFunc("GET /v1/model-pricing", h.auth(h.handleListPricing))
|
||||
mux.HandleFunc("PUT /v1/model-pricing/overrides", h.writeAuth(h.handlePutOverride))
|
||||
mux.HandleFunc("GET /v1/model-pricing/overrides", h.auth(h.handleListOverrides))
|
||||
mux.HandleFunc("DELETE /v1/model-pricing/overrides/{id}", h.writeAuth(h.handleDeleteOverride))
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) auth(next http.HandlerFunc) http.HandlerFunc {
|
||||
return requireAuth(permissions.RoleAdmin, next)
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) writeAuth(next http.HandlerFunc) http.HandlerFunc {
|
||||
return h.auth(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireTenantAdmin(w, r, h.tenants) {
|
||||
return
|
||||
}
|
||||
next(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) masterAuth(next http.HandlerFunc) http.HandlerFunc {
|
||||
return h.auth(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !requireMasterScope(w, r) {
|
||||
return
|
||||
}
|
||||
next(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func writeUsageCapError(w http.ResponseWriter, r *http.Request, status int, key string, args ...any) {
|
||||
writeJSON(w, status, map[string]string{"error": i18n.T(store.LocaleFromContext(r.Context()), key, args...)})
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handleListPolicies(w http.ResponseWriter, r *http.Request) {
|
||||
scope := store.UsageCapScope{TenantID: tenantIDOrMaster(r)}
|
||||
policies, err := h.store.ListUsageCapPolicies(r.Context(), scope, true)
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsListPoliciesFailed)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"policies": policies})
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handleCreatePolicy(w http.ResponseWriter, r *http.Request) {
|
||||
var body policyBody
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidJSON)
|
||||
return
|
||||
}
|
||||
p, err := body.toPolicy(tenantIDOrMaster(r))
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if err := h.store.CreateUsageCapPolicy(r.Context(), &p); err != nil {
|
||||
slog.Warn("usage_caps.create_policy_failed", "error", err)
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsageCapPolicyValidationFailed)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, p)
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handleUpdatePolicy(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := uuid.Parse(r.PathValue("id"))
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "policy")
|
||||
return
|
||||
}
|
||||
bodyBytes, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidJSON)
|
||||
return
|
||||
}
|
||||
patch, err := policyPatchFromBody(bodyBytes)
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidRequest, err.Error())
|
||||
return
|
||||
}
|
||||
p, err := h.store.UpdateUsageCapPolicy(r.Context(), tenantIDOrMaster(r), id, patch)
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrUsageCapPolicyManaged) {
|
||||
writeUsageCapError(w, r, http.StatusConflict, i18n.MsgUsageCapPolicyManaged)
|
||||
return
|
||||
}
|
||||
slog.Warn("usage_caps.update_policy_failed", "error", err)
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsageCapPolicyValidationFailed)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, p)
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handleDeletePolicy(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := uuid.Parse(r.PathValue("id"))
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "policy")
|
||||
return
|
||||
}
|
||||
if err := h.store.DeleteUsageCapPolicy(r.Context(), tenantIDOrMaster(r), id); err != nil {
|
||||
if errors.Is(err, store.ErrUsageCapPolicyManaged) {
|
||||
writeUsageCapError(w, r, http.StatusConflict, i18n.MsgUsageCapPolicyManaged)
|
||||
return
|
||||
}
|
||||
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsDeletePolicyFailed)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handleUtilization(w http.ResponseWriter, r *http.Request) {
|
||||
rows, err := h.store.ListUsageCapUtilization(r.Context(), tenantIDOrMaster(r))
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsUtilizationFailed)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"rows": rows})
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handleEvents(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
events, err := h.store.ListUsageCapEvents(r.Context(), tenantIDOrMaster(r), limit)
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsageCapsEventsFailed)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"events": events})
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handleSyncOpenRouter(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
|
||||
defer cancel()
|
||||
entries, err := pricing.FetchOpenRouterCatalog(ctx, h.client)
|
||||
if err != nil {
|
||||
slog.Warn("usage_pricing.openrouter_sync", "error", err)
|
||||
writeUsageCapError(w, r, http.StatusBadGateway, i18n.MsgUsagePricingSyncOpenRouterFailed, err.Error())
|
||||
return
|
||||
}
|
||||
count, err := h.store.UpsertPricingCatalog(r.Context(), entries)
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingStoreCatalogFailed)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"count": count})
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handleListPricing(w http.ResponseWriter, r *http.Request) {
|
||||
rows, err := h.store.ListPricingCatalog(r.Context(), store.UsagePricingQuery{
|
||||
ModelID: r.URL.Query().Get("model"),
|
||||
Limit: queryInt(r, "limit", 100),
|
||||
})
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingListFailed)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"models": rows})
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handlePutOverride(w http.ResponseWriter, r *http.Request) {
|
||||
var body overrideBody
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidJSON)
|
||||
return
|
||||
}
|
||||
providerID, err := uuid.Parse(body.ProviderID)
|
||||
if err != nil || body.ModelID == "" {
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsagePricingProviderModelRequired)
|
||||
return
|
||||
}
|
||||
o := &store.UsagePricingOverride{
|
||||
TenantID: tenantIDOrMaster(r), ProviderID: providerID,
|
||||
ProviderType: body.ProviderType, ModelID: body.ModelID,
|
||||
Pricing: body.Pricing, Enabled: body.Enabled == nil || *body.Enabled,
|
||||
}
|
||||
if err := h.store.PutPricingOverride(r.Context(), o); err != nil {
|
||||
slog.Warn("usage_pricing.put_override_failed", "error", err)
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgUsagePricingOverrideValidationFailed)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, o)
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handleListOverrides(w http.ResponseWriter, r *http.Request) {
|
||||
var providerID uuid.UUID
|
||||
if raw := r.URL.Query().Get("provider_id"); raw != "" {
|
||||
var err error
|
||||
providerID, err = uuid.Parse(raw)
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "provider")
|
||||
return
|
||||
}
|
||||
}
|
||||
rows, err := h.store.ListPricingOverrides(r.Context(), store.UsagePricingQuery{TenantID: tenantIDOrMaster(r), ProviderID: providerID})
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingListOverridesFailed)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"overrides": rows})
|
||||
}
|
||||
|
||||
func (h *UsageCapsHandler) handleDeleteOverride(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := uuid.Parse(r.PathValue("id"))
|
||||
if err != nil {
|
||||
writeUsageCapError(w, r, http.StatusBadRequest, i18n.MsgInvalidID, "override")
|
||||
return
|
||||
}
|
||||
if err := h.store.DeletePricingOverride(r.Context(), tenantIDOrMaster(r), id); err != nil {
|
||||
writeUsageCapError(w, r, http.StatusInternalServerError, i18n.MsgUsagePricingDeleteOverrideFailed)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
type policyBody struct {
|
||||
AgentID *string `json:"agent_id"`
|
||||
ProviderID *string `json:"provider_id"`
|
||||
ProviderType *string `json:"provider_type"`
|
||||
ModelID *string `json:"model_id"`
|
||||
Window string `json:"window"`
|
||||
MaxTokens *int64 `json:"max_tokens"`
|
||||
MaxCostMicros *int64 `json:"max_cost_micros"`
|
||||
MaxCostUSD *float64 `json:"max_cost_usd"`
|
||||
Enabled *bool `json:"enabled"`
|
||||
Priority *int `json:"priority"`
|
||||
}
|
||||
|
||||
type overrideBody struct {
|
||||
ProviderID string `json:"provider_id"`
|
||||
ProviderType string `json:"provider_type"`
|
||||
ModelID string `json:"model_id"`
|
||||
Pricing store.UsagePricingFields `json:"pricing"`
|
||||
Enabled *bool `json:"enabled"`
|
||||
}
|
||||
|
||||
func (b policyBody) toPolicy(tenantID uuid.UUID) (store.UsageCapPolicy, error) {
|
||||
p := store.UsageCapPolicy{TenantID: tenantID, Window: b.Window, Enabled: true, Priority: 100}
|
||||
if b.Enabled != nil {
|
||||
p.Enabled = *b.Enabled
|
||||
}
|
||||
if b.Priority != nil {
|
||||
p.Priority = *b.Priority
|
||||
}
|
||||
if p.Window == "" {
|
||||
p.Window = store.UsageCapWindowDay
|
||||
}
|
||||
var err error
|
||||
p.AgentID, err = parseOptionalUUIDStrict("agent_id", b.AgentID)
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
p.ProviderID, err = parseOptionalUUIDStrict("provider_id", b.ProviderID)
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
if b.ProviderType != nil {
|
||||
p.ProviderType = *b.ProviderType
|
||||
}
|
||||
if b.ModelID != nil {
|
||||
p.ModelID = *b.ModelID
|
||||
}
|
||||
p.MaxTokens = b.MaxTokens
|
||||
p.MaxCostMicros = maxCostMicros(b.MaxCostMicros, b.MaxCostUSD)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (b policyBody) toPatch() (store.UsageCapPolicyPatch, error) {
|
||||
var patch store.UsageCapPolicyPatch
|
||||
if b.AgentID != nil {
|
||||
v, err := parseOptionalUUIDStrict("agent_id", b.AgentID)
|
||||
if err != nil {
|
||||
return patch, err
|
||||
}
|
||||
patch.AgentID = &v
|
||||
}
|
||||
if b.ProviderID != nil {
|
||||
v, err := parseOptionalUUIDStrict("provider_id", b.ProviderID)
|
||||
if err != nil {
|
||||
return patch, err
|
||||
}
|
||||
patch.ProviderID = &v
|
||||
}
|
||||
patch.ProviderType = b.ProviderType
|
||||
patch.ModelID = b.ModelID
|
||||
if b.Window != "" {
|
||||
patch.Window = &b.Window
|
||||
}
|
||||
if b.MaxTokens != nil {
|
||||
patch.MaxTokens = &b.MaxTokens
|
||||
}
|
||||
if b.MaxCostMicros != nil || b.MaxCostUSD != nil {
|
||||
v := maxCostMicros(b.MaxCostMicros, b.MaxCostUSD)
|
||||
patch.MaxCostMicros = &v
|
||||
}
|
||||
patch.Enabled = b.Enabled
|
||||
patch.Priority = b.Priority
|
||||
return patch, nil
|
||||
}
|
||||
|
||||
func policyPatchFromBody(bodyBytes []byte) (store.UsageCapPolicyPatch, error) {
|
||||
var body policyBody
|
||||
if err := json.Unmarshal(bodyBytes, &body); err != nil {
|
||||
return store.UsageCapPolicyPatch{}, err
|
||||
}
|
||||
patch, err := body.toPatch()
|
||||
if err != nil {
|
||||
return patch, err
|
||||
}
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(bodyBytes, &raw); err == nil {
|
||||
if isJSONNull(raw["max_tokens"]) {
|
||||
var v *int64
|
||||
patch.MaxTokens = &v
|
||||
}
|
||||
if isJSONNull(raw["max_cost_micros"]) || isJSONNull(raw["max_cost_usd"]) {
|
||||
var v *int64
|
||||
patch.MaxCostMicros = &v
|
||||
}
|
||||
}
|
||||
return patch, nil
|
||||
}
|
||||
|
||||
func isJSONNull(raw json.RawMessage) bool {
|
||||
return strings.TrimSpace(string(raw)) == "null"
|
||||
}
|
||||
|
||||
func tenantIDOrMaster(r *http.Request) uuid.UUID {
|
||||
if tid := store.TenantIDFromContext(r.Context()); tid != uuid.Nil {
|
||||
return tid
|
||||
}
|
||||
return store.MasterTenantID
|
||||
}
|
||||
|
||||
func queryInt(r *http.Request, key string, fallback int) int {
|
||||
n, err := strconv.Atoi(r.URL.Query().Get(key))
|
||||
if err != nil || n <= 0 {
|
||||
return fallback
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func parseOptionalUUIDStrict(field string, raw *string) (*uuid.UUID, error) {
|
||||
if raw == nil || *raw == "" {
|
||||
return nil, nil
|
||||
}
|
||||
id, err := uuid.Parse(*raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid %s", field)
|
||||
}
|
||||
return &id, nil
|
||||
}
|
||||
|
||||
func maxCostMicros(micros *int64, usd *float64) *int64 {
|
||||
if micros != nil {
|
||||
return micros
|
||||
}
|
||||
if usd == nil {
|
||||
return nil
|
||||
}
|
||||
v := int64(*usd * 1_000_000)
|
||||
return &v
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package http
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestPolicyPatchFromBodyClearsTokenAndCostLimits(t *testing.T) {
|
||||
patch, err := policyPatchFromBody([]byte(`{"max_tokens":null,"max_cost_usd":null}`))
|
||||
if err != nil {
|
||||
t.Fatalf("policyPatchFromBody: %v", err)
|
||||
}
|
||||
if patch.MaxTokens == nil {
|
||||
t.Fatal("MaxTokens patch missing")
|
||||
}
|
||||
if *patch.MaxTokens != nil {
|
||||
t.Fatalf("MaxTokens = %v, want nil clear", **patch.MaxTokens)
|
||||
}
|
||||
if patch.MaxCostMicros == nil {
|
||||
t.Fatal("MaxCostMicros patch missing")
|
||||
}
|
||||
if *patch.MaxCostMicros != nil {
|
||||
t.Fatalf("MaxCostMicros = %v, want nil clear", **patch.MaxCostMicros)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolicyPatchFromBodyPreservesProvidedCostUSD(t *testing.T) {
|
||||
patch, err := policyPatchFromBody([]byte(`{"max_cost_usd":12.5}`))
|
||||
if err != nil {
|
||||
t.Fatalf("policyPatchFromBody: %v", err)
|
||||
}
|
||||
if patch.MaxCostMicros == nil || *patch.MaxCostMicros == nil {
|
||||
t.Fatal("MaxCostMicros patch missing")
|
||||
}
|
||||
if got := **patch.MaxCostMicros; got != 12_500_000 {
|
||||
t.Fatalf("MaxCostMicros = %d, want 12500000", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func TestUsageCapPolicyBodyRejectsInvalidUUID(t *testing.T) {
|
||||
invalid := "not-a-uuid"
|
||||
if _, err := (policyBody{AgentID: &invalid}).toPolicy(uuid.New()); err == nil {
|
||||
t.Fatal("toPolicy accepted invalid agent_id")
|
||||
}
|
||||
if _, err := (policyBody{ProviderID: &invalid}).toPatch(); err == nil {
|
||||
t.Fatal("toPatch accepted invalid provider_id")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user