From b284f963f5183fd94e8aee525b719b0bb2f3181f Mon Sep 17 00:00:00 2001 From: viettranx Date: Mon, 9 Mar 2026 14:54:41 +0700 Subject: [PATCH] feat(memory): add memory management page with CRUD, search, and indexing Add full-stack memory document management: - Backend: extend MemoryStore with admin queries (ListAllDocumentsGlobal, GetDocumentDetail, ListChunks), HTTP handler with auth middleware - Frontend: memory page with agent/scope filters, document table with pagination, view/edit dialog with content and chunks tabs, create dialog with scope selection, semantic search dialog - UI fixes: reduce input/textarea focus ring width, prevent ring clipping in dialog scroll containers, widen memory dialogs on desktop --- CLAUDE.md | 1 + cmd/gateway.go | 5 + internal/gateway/server.go | 9 + internal/http/memory.go | 43 +++ internal/http/memory_handlers.go | 190 +++++++++++ internal/store/memory_store.go | 28 ++ internal/store/pg/memory_admin.go | 163 ++++++++++ ui/web/src/components/layout/sidebar.tsx | 2 + ui/web/src/components/ui/input.tsx | 2 +- ui/web/src/components/ui/textarea.tsx | 2 +- ui/web/src/lib/constants.ts | 1 + ui/web/src/lib/query-keys.ts | 4 + ui/web/src/pages/memory/hooks/use-memory.ts | 201 ++++++++++++ .../src/pages/memory/memory-create-dialog.tsx | 236 ++++++++++++++ .../pages/memory/memory-document-dialog.tsx | 203 ++++++++++++ ui/web/src/pages/memory/memory-page.tsx | 305 ++++++++++++++++++ .../src/pages/memory/memory-search-dialog.tsx | 114 +++++++ ui/web/src/routes.tsx | 4 + ui/web/src/types/memory.ts | 35 ++ 19 files changed, 1546 insertions(+), 2 deletions(-) create mode 100644 internal/http/memory.go create mode 100644 internal/http/memory_handlers.go create mode 100644 internal/store/pg/memory_admin.go create mode 100644 ui/web/src/pages/memory/hooks/use-memory.ts create mode 100644 ui/web/src/pages/memory/memory-create-dialog.tsx create mode 100644 ui/web/src/pages/memory/memory-document-dialog.tsx create mode 100644 ui/web/src/pages/memory/memory-page.tsx create mode 100644 ui/web/src/pages/memory/memory-search-dialog.tsx create mode 100644 ui/web/src/types/memory.ts diff --git a/CLAUDE.md b/CLAUDE.md index efb3da4a..7b5a54d2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -77,3 +77,4 @@ Go conventions to follow: - Use `switch/case` instead of `if/else if` chains on the same variable - Use `append(dst, src...)` instead of loop-based append - Always handle errors; don't ignore return values +- **Migrations:** When adding a new SQL migration file in `migrations/`, bump `RequiredSchemaVersion` in `internal/upgrade/version.go` to match the new migration number diff --git a/cmd/gateway.go b/cmd/gateway.go index 755dbfed..e35ecc37 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -617,6 +617,11 @@ func runGateway() { server.SetPendingMessagesHandler(pendingMessagesH) } + // Memory management API (wired directly, only needs MemoryStore + token) + if pgStores != nil && pgStores.Memory != nil { + server.SetMemoryHandler(httpapi.NewMemoryHandler(pgStores.Memory, cfg.Gateway.Token)) + } + // Workspace file serving endpoint — serves files by absolute path, auth-token protected. // Supports media from any agent workspace (each agent has its own workspace from DB). server.SetFilesHandler(httpapi.NewFilesHandler(cfg.Gateway.Token)) diff --git a/internal/gateway/server.go b/internal/gateway/server.go index c4f18a58..1a8e551b 100644 --- a/internal/gateway/server.go +++ b/internal/gateway/server.go @@ -45,6 +45,7 @@ type Server struct { delegationsHandler *httpapi.DelegationsHandler // delegation history API builtinToolsHandler *httpapi.BuiltinToolsHandler // builtin tool management API pendingMessagesHandler *httpapi.PendingMessagesHandler // pending messages API + memoryHandler *httpapi.MemoryHandler // memory management API oauthHandler *httpapi.OAuthHandler // OAuth endpoints filesHandler *httpapi.FilesHandler // workspace file serving storageHandler *httpapi.StorageHandler // storage file management @@ -205,6 +206,11 @@ func (s *Server) BuildMux() *http.ServeMux { s.pendingMessagesHandler.RegisterRoutes(mux) } + // Memory management API + if s.memoryHandler != nil { + s.memoryHandler.RegisterRoutes(mux) + } + // Workspace file serving (available in all modes) if s.filesHandler != nil { s.filesHandler.RegisterRoutes(mux) @@ -385,6 +391,9 @@ func (s *Server) SetMediaUploadHandler(h *httpapi.MediaUploadHandler) { s.mediaU // SetMediaServeHandler sets the media serve handler. func (s *Server) SetMediaServeHandler(h *httpapi.MediaServeHandler) { s.mediaServeHandler = h } +// SetMemoryHandler sets the memory management handler. +func (s *Server) SetMemoryHandler(h *httpapi.MemoryHandler) { s.memoryHandler = h } + // SetAgentStore sets the agent store for context injection in tools_invoke. func (s *Server) SetAgentStore(as store.AgentStore) { s.agentStore = as } diff --git a/internal/http/memory.go b/internal/http/memory.go new file mode 100644 index 00000000..79fdfd10 --- /dev/null +++ b/internal/http/memory.go @@ -0,0 +1,43 @@ +package http + +import ( + "net/http" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// MemoryHandler handles memory document management endpoints. +type MemoryHandler struct { + store store.MemoryStore + token string +} + +// NewMemoryHandler creates a handler for memory management endpoints. +func NewMemoryHandler(s store.MemoryStore, token string) *MemoryHandler { + return &MemoryHandler{store: s, token: token} +} + +// RegisterRoutes registers all memory routes on the given mux. +func (h *MemoryHandler) RegisterRoutes(mux *http.ServeMux) { + mux.HandleFunc("GET /v1/memory/documents", h.auth(h.handleListAllDocuments)) + mux.HandleFunc("GET /v1/agents/{agentID}/memory/documents", h.auth(h.handleListDocuments)) + mux.HandleFunc("GET /v1/agents/{agentID}/memory/documents/{path...}", h.auth(h.handleGetDocument)) + mux.HandleFunc("PUT /v1/agents/{agentID}/memory/documents/{path...}", h.auth(h.handlePutDocument)) + mux.HandleFunc("DELETE /v1/agents/{agentID}/memory/documents/{path...}", h.auth(h.handleDeleteDocument)) + mux.HandleFunc("GET /v1/agents/{agentID}/memory/chunks", h.auth(h.handleListChunks)) + mux.HandleFunc("POST /v1/agents/{agentID}/memory/index", h.auth(h.handleIndexDocument)) + mux.HandleFunc("POST /v1/agents/{agentID}/memory/index-all", h.auth(h.handleIndexAll)) + mux.HandleFunc("POST /v1/agents/{agentID}/memory/search", h.auth(h.handleSearch)) +} + +func (h *MemoryHandler) auth(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if h.token != "" { + if extractBearerToken(r) != h.token { + writeJSON(w, http.StatusUnauthorized, map[string]string{"error": "unauthorized"}) + return + } + } + next(w, r) + } +} diff --git a/internal/http/memory_handlers.go b/internal/http/memory_handlers.go new file mode 100644 index 00000000..2b6d63ed --- /dev/null +++ b/internal/http/memory_handlers.go @@ -0,0 +1,190 @@ +package http + +import ( + "encoding/json" + "log/slog" + "net/http" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +func (h *MemoryHandler) handleListAllDocuments(w http.ResponseWriter, r *http.Request) { + docs, err := h.store.ListAllDocumentsGlobal(r.Context()) + if err != nil { + slog.Warn("memory.list_all_documents failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if docs == nil { + docs = []store.DocumentInfo{} + } + writeJSON(w, http.StatusOK, docs) +} + +func (h *MemoryHandler) handleListDocuments(w http.ResponseWriter, r *http.Request) { + agentID := r.PathValue("agentID") + userID := r.URL.Query().Get("user_id") + + var docs []store.DocumentInfo + var err error + if userID == "" { + docs, err = h.store.ListAllDocuments(r.Context(), agentID) + } else { + docs, err = h.store.ListDocuments(r.Context(), agentID, userID) + } + if err != nil { + slog.Warn("memory.list_documents failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if docs == nil { + docs = []store.DocumentInfo{} + } + writeJSON(w, http.StatusOK, docs) +} + +func (h *MemoryHandler) handleGetDocument(w http.ResponseWriter, r *http.Request) { + agentID := r.PathValue("agentID") + path := r.PathValue("path") + userID := r.URL.Query().Get("user_id") + + detail, err := h.store.GetDocumentDetail(r.Context(), agentID, userID, path) + if err != nil { + slog.Warn("memory.get_document failed", "error", err, "path", path) + writeJSON(w, http.StatusNotFound, map[string]string{"error": "document not found"}) + return + } + writeJSON(w, http.StatusOK, detail) +} + +func (h *MemoryHandler) handlePutDocument(w http.ResponseWriter, r *http.Request) { + agentID := r.PathValue("agentID") + path := r.PathValue("path") + + var body struct { + Content string `json:"content"` + UserID string `json:"user_id"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + + if err := h.store.PutDocument(r.Context(), agentID, body.UserID, path, body.Content); err != nil { + slog.Warn("memory.put_document failed", "error", err, "path", path) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok", "path": path}) +} + +func (h *MemoryHandler) handleDeleteDocument(w http.ResponseWriter, r *http.Request) { + agentID := r.PathValue("agentID") + path := r.PathValue("path") + userID := r.URL.Query().Get("user_id") + + if err := h.store.DeleteDocument(r.Context(), agentID, userID, path); err != nil { + slog.Warn("memory.delete_document failed", "error", err, "path", path) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "deleted"}) +} + +func (h *MemoryHandler) handleListChunks(w http.ResponseWriter, r *http.Request) { + agentID := r.PathValue("agentID") + path := r.URL.Query().Get("path") + userID := r.URL.Query().Get("user_id") + + if path == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "path query parameter required"}) + return + } + + chunks, err := h.store.ListChunks(r.Context(), agentID, userID, path) + if err != nil { + slog.Warn("memory.list_chunks failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if chunks == nil { + chunks = []store.ChunkInfo{} + } + writeJSON(w, http.StatusOK, chunks) +} + +func (h *MemoryHandler) handleIndexDocument(w http.ResponseWriter, r *http.Request) { + agentID := r.PathValue("agentID") + + var body struct { + Path string `json:"path"` + UserID string `json:"user_id"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + if body.Path == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "path is required"}) + return + } + + if err := h.store.IndexDocument(r.Context(), agentID, body.UserID, body.Path); err != nil { + slog.Warn("memory.index_document failed", "error", err, "path", body.Path) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "indexed", "path": body.Path}) +} + +func (h *MemoryHandler) handleIndexAll(w http.ResponseWriter, r *http.Request) { + agentID := r.PathValue("agentID") + + var body struct { + UserID string `json:"user_id"` + } + json.NewDecoder(r.Body).Decode(&body) + + if err := h.store.IndexAll(r.Context(), agentID, body.UserID); err != nil { + slog.Warn("memory.index_all failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "indexed_all"}) +} + +func (h *MemoryHandler) handleSearch(w http.ResponseWriter, r *http.Request) { + agentID := r.PathValue("agentID") + + var body struct { + Query string `json:"query"` + UserID string `json:"user_id"` + MaxResults int `json:"max_results"` + MinScore float64 `json:"min_score"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid request body"}) + return + } + if body.Query == "" { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "query is required"}) + return + } + + results, err := h.store.Search(r.Context(), body.Query, agentID, body.UserID, store.MemorySearchOptions{ + MaxResults: body.MaxResults, + MinScore: body.MinScore, + }) + if err != nil { + slog.Warn("memory.search failed", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) + return + } + if results == nil { + results = []store.MemorySearchResult{} + } + writeJSON(w, http.StatusOK, map[string]interface{}{ + "results": results, + "count": len(results), + }) +} diff --git a/internal/store/memory_store.go b/internal/store/memory_store.go index 64772259..cefbfb3c 100644 --- a/internal/store/memory_store.go +++ b/internal/store/memory_store.go @@ -6,6 +6,7 @@ import "context" type DocumentInfo struct { Path string `json:"path"` Hash string `json:"hash"` + AgentID string `json:"agent_id,omitempty"` UserID string `json:"user_id,omitempty"` UpdatedAt int64 `json:"updated_at"` } @@ -36,6 +37,27 @@ type EmbeddingProvider interface { Embed(ctx context.Context, texts []string) ([][]float32, error) } +// DocumentDetail provides full document info including chunk/embedding stats. +type DocumentDetail struct { + Path string `json:"path"` + Content string `json:"content"` + Hash string `json:"hash"` + UserID string `json:"user_id,omitempty"` + ChunkCount int `json:"chunk_count"` + EmbeddedCount int `json:"embedded_count"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +// ChunkInfo describes a single memory chunk. +type ChunkInfo struct { + ID string `json:"id"` + StartLine int `json:"start_line"` + EndLine int `json:"end_line"` + TextPreview string `json:"text_preview"` + HasEmbedding bool `json:"has_embedding"` +} + // MemoryStore manages memory documents and search. type MemoryStore interface { // Document CRUD @@ -44,6 +66,12 @@ type MemoryStore interface { DeleteDocument(ctx context.Context, agentID, userID, path string) error ListDocuments(ctx context.Context, agentID, userID string) ([]DocumentInfo, error) + // Admin queries + ListAllDocumentsGlobal(ctx context.Context) ([]DocumentInfo, error) + ListAllDocuments(ctx context.Context, agentID string) ([]DocumentInfo, error) + GetDocumentDetail(ctx context.Context, agentID, userID, path string) (*DocumentDetail, error) + ListChunks(ctx context.Context, agentID, userID, path string) ([]ChunkInfo, error) + // Search Search(ctx context.Context, query string, agentID, userID string, opts MemorySearchOptions) ([]MemorySearchResult, error) diff --git a/internal/store/pg/memory_admin.go b/internal/store/pg/memory_admin.go new file mode 100644 index 00000000..b178ea08 --- /dev/null +++ b/internal/store/pg/memory_admin.go @@ -0,0 +1,163 @@ +package pg + +import ( + "context" + "time" + + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// ListAllDocumentsGlobal returns all documents across all agents (for admin overview). +func (s *PGMemoryStore) ListAllDocumentsGlobal(ctx context.Context) ([]store.DocumentInfo, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT agent_id, path, hash, user_id, updated_at + FROM memory_documents + ORDER BY updated_at DESC`) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []store.DocumentInfo + for rows.Next() { + var agentID, path, hash string + var uid *string + var updatedAt time.Time + if err := rows.Scan(&agentID, &path, &hash, &uid, &updatedAt); err != nil { + continue + } + info := store.DocumentInfo{ + AgentID: agentID, + Path: path, + Hash: hash, + UpdatedAt: updatedAt.UnixMilli(), + } + if uid != nil { + info.UserID = *uid + } + result = append(result, info) + } + return result, nil +} + +// ListAllDocuments returns all documents for an agent across all users (global + personal). +func (s *PGMemoryStore) ListAllDocuments(ctx context.Context, agentID string) ([]store.DocumentInfo, error) { + aid := mustParseUUID(agentID) + rows, err := s.db.QueryContext(ctx, + `SELECT agent_id, path, hash, user_id, updated_at + FROM memory_documents WHERE agent_id = $1 + ORDER BY updated_at DESC`, aid) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []store.DocumentInfo + for rows.Next() { + var aID, path, hash string + var uid *string + var updatedAt time.Time + if err := rows.Scan(&aID, &path, &hash, &uid, &updatedAt); err != nil { + continue + } + info := store.DocumentInfo{ + AgentID: aID, + Path: path, + Hash: hash, + UpdatedAt: updatedAt.UnixMilli(), + } + if uid != nil { + info.UserID = *uid + } + result = append(result, info) + } + return result, nil +} + +// GetDocumentDetail returns full document info with chunk and embedding counts. +func (s *PGMemoryStore) GetDocumentDetail(ctx context.Context, agentID, userID, path string) (*store.DocumentDetail, error) { + aid := mustParseUUID(agentID) + + var q string + var args []interface{} + if userID == "" { + q = `SELECT d.path, d.content, d.hash, d.user_id, d.created_at, d.updated_at, + COUNT(c.id) AS chunk_count, + COUNT(c.embedding) AS embedded_count + FROM memory_documents d + LEFT JOIN memory_chunks c ON c.document_id = d.id + WHERE d.agent_id = $1 AND d.path = $2 AND d.user_id IS NULL + GROUP BY d.id` + args = []interface{}{aid, path} + } else { + q = `SELECT d.path, d.content, d.hash, d.user_id, d.created_at, d.updated_at, + COUNT(c.id) AS chunk_count, + COUNT(c.embedding) AS embedded_count + FROM memory_documents d + LEFT JOIN memory_chunks c ON c.document_id = d.id + WHERE d.agent_id = $1 AND d.path = $2 AND d.user_id = $3 + GROUP BY d.id` + args = []interface{}{aid, path, userID} + } + + var detail store.DocumentDetail + var uid *string + var createdAt, updatedAt time.Time + err := s.db.QueryRowContext(ctx, q, args...).Scan( + &detail.Path, &detail.Content, &detail.Hash, &uid, + &createdAt, &updatedAt, + &detail.ChunkCount, &detail.EmbeddedCount, + ) + if err != nil { + return nil, err + } + if uid != nil { + detail.UserID = *uid + } + detail.CreatedAt = createdAt.UnixMilli() + detail.UpdatedAt = updatedAt.UnixMilli() + return &detail, nil +} + +// ListChunks returns chunks for a document identified by agent, user, and path. +func (s *PGMemoryStore) ListChunks(ctx context.Context, agentID, userID, path string) ([]store.ChunkInfo, error) { + aid := mustParseUUID(agentID) + + var q string + var args []interface{} + if userID == "" { + q = `SELECT c.id, c.start_line, c.end_line, + LEFT(c.text, 200) AS text_preview, + (c.embedding IS NOT NULL) AS has_embedding + FROM memory_chunks c + JOIN memory_documents d ON c.document_id = d.id + WHERE d.agent_id = $1 AND d.path = $2 AND d.user_id IS NULL + ORDER BY c.start_line` + args = []interface{}{aid, path} + } else { + q = `SELECT c.id, c.start_line, c.end_line, + LEFT(c.text, 200) AS text_preview, + (c.embedding IS NOT NULL) AS has_embedding + FROM memory_chunks c + JOIN memory_documents d ON c.document_id = d.id + WHERE d.agent_id = $1 AND d.path = $2 AND d.user_id = $3 + ORDER BY c.start_line` + args = []interface{}{aid, path, userID} + } + + rows, err := s.db.QueryContext(ctx, q, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []store.ChunkInfo + for rows.Next() { + var ci store.ChunkInfo + if err := rows.Scan(&ci.ID, &ci.StartLine, &ci.EndLine, &ci.TextPreview, &ci.HasEmbedding); err != nil { + continue + } + result = append(result, ci) + } + return result, nil +} diff --git a/ui/web/src/components/layout/sidebar.tsx b/ui/web/src/components/layout/sidebar.tsx index c0281ffd..e114784d 100644 --- a/ui/web/src/components/layout/sidebar.tsx +++ b/ui/web/src/components/layout/sidebar.tsx @@ -22,6 +22,7 @@ import { ArrowRightLeft, HardDrive, Inbox, + Brain, } from "lucide-react"; import { SidebarGroup } from "./sidebar-group"; import { SidebarItem } from "./sidebar-item"; @@ -81,6 +82,7 @@ export function Sidebar({ collapsed, onNavItemClick }: SidebarProps) { + diff --git a/ui/web/src/components/ui/input.tsx b/ui/web/src/components/ui/input.tsx index f0b32a56..c4b77ded 100644 --- a/ui/web/src/components/ui/input.tsx +++ b/ui/web/src/components/ui/input.tsx @@ -9,7 +9,7 @@ function Input({ className, type, ...props }: React.ComponentProps<"input">) { data-slot="input" className={cn( "file:text-foreground placeholder:text-muted-foreground selection:bg-primary selection:text-primary-foreground dark:bg-input/30 border-input h-9 w-full min-w-0 rounded-md border bg-transparent px-3 py-1 text-base shadow-xs transition-[color,box-shadow] outline-none file:inline-flex file:h-7 file:border-0 file:bg-transparent file:text-sm file:font-medium disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50 md:text-sm", - "focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-2", + "focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-1", "aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive", className )} diff --git a/ui/web/src/components/ui/textarea.tsx b/ui/web/src/components/ui/textarea.tsx index 7f21b5e7..210bfc56 100644 --- a/ui/web/src/components/ui/textarea.tsx +++ b/ui/web/src/components/ui/textarea.tsx @@ -7,7 +7,7 @@ function Textarea({ className, ...props }: React.ComponentProps<"textarea">) {