package http import ( "context" "encoding/json" "fmt" "log/slog" "strings" "github.com/google/uuid" "github.com/nextlevelbuilder/goclaw/internal/config" "github.com/nextlevelbuilder/goclaw/internal/store" ) // doMergeImport upserts sections of the archive into the target agent. func (h *AgentsHandler) doMergeImport(ctx context.Context, ag *store.AgentData, arc *importArchive, sections map[string]bool, progressFn func(ProgressEvent)) (*ImportSummary, error) { summary := &ImportSummary{AgentID: ag.ID.String(), AgentKey: ag.AgentKey} // Section: context_files if sections["context_files"] { if err := h.importContextFiles(ctx, ag, arc, summary, progressFn); err != nil { return nil, err } } // Section: memory if sections["memory"] && h.memoryStore != nil { if err := h.importMemory(ctx, ag, arc, summary, progressFn); err != nil { return nil, err } } // Section: knowledge_graph if sections["knowledge_graph"] && h.kgStore != nil && len(arc.kgEntities) > 0 { if err := h.importKG(ctx, ag, arc, summary, progressFn); err != nil { return nil, err } } // Section: cron — always imported as disabled if sections["cron"] && len(arc.cronJobs) > 0 { h.importCron(ctx, ag, arc, summary, progressFn) } // Section: user_profiles if sections["user_profiles"] && len(arc.userProfiles) > 0 { h.importUserProfiles(ctx, ag, arc, summary, progressFn) } // Section: user_overrides if sections["user_overrides"] && len(arc.userOverrides) > 0 { h.importUserOverrides(ctx, ag, arc, summary, progressFn) } // Section: workspace files if sections["workspace"] && len(arc.workspaceFiles) > 0 { h.importWorkspace(ctx, ag, arc, summary, progressFn) } // Section: team if sections["team"] && arc.teamMeta != nil { if err := h.importTeamSection(ctx, ag, arc, progressFn); err != nil { slog.Warn("import: team section failed", "agent_id", ag.ID, "error", err) } else { summary.TeamImported = true } } // Section: evolution metrics + suggestions if sections["evolution"] { h.importEvolution(ctx, ag, arc, summary, progressFn) } // Section: episodic summaries (Tier 2 memory) — PG only, nil-guarded if sections["episodic"] && h.episodicStore != nil && len(arc.episodicSummaries) > 0 { h.importEpisodic(ctx, ag, arc, summary, progressFn) } // Section: vault (Knowledge Vault documents + links) if sections["vault"] && h.vaultStore != nil && len(arc.vaultDocuments) > 0 { h.importVault(ctx, ag, arc, summary, progressFn) } return summary, nil } func (h *AgentsHandler) importContextFiles(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) error { if progressFn != nil { progressFn(ProgressEvent{Phase: "context_files", Status: "running", Total: len(arc.contextFiles)}) } for _, cf := range arc.contextFiles { if err := h.agents.SetAgentContextFile(ctx, ag.ID, cf.fileName, cf.content); err != nil { return fmt.Errorf("set context file %s: %w", cf.fileName, err) } summary.ContextFiles++ } for _, ucf := range arc.userContextFiles { if err := h.agents.SetUserContextFile(ctx, ag.ID, ucf.userID, ucf.fileName, ucf.content); err != nil { return fmt.Errorf("set user context file %s/%s: %w", ucf.userID, ucf.fileName, err) } summary.UserContextFiles++ } if progressFn != nil { total := summary.ContextFiles + summary.UserContextFiles progressFn(ProgressEvent{Phase: "context_files", Status: "done", Current: total, Total: total}) } return nil } func (h *AgentsHandler) importMemory(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) error { totalDocs := len(arc.memoryGlobal) + countUserMemory(arc.memoryUsers) if progressFn != nil { progressFn(ProgressEvent{Phase: "memory", Status: "running", Total: totalDocs}) } for _, doc := range arc.memoryGlobal { if err := h.memoryStore.PutDocument(ctx, ag.ID.String(), "", doc.Path, doc.Content); err != nil { return fmt.Errorf("put memory doc %s: %w", doc.Path, err) } summary.MemoryDocs++ } for uid, docs := range arc.memoryUsers { for _, doc := range docs { if err := h.memoryStore.PutDocument(ctx, ag.ID.String(), uid, doc.Path, doc.Content); err != nil { return fmt.Errorf("put memory doc %s/%s: %w", uid, doc.Path, err) } summary.MemoryDocs++ } } if progressFn != nil { progressFn(ProgressEvent{Phase: "memory", Status: "done", Current: summary.MemoryDocs, Total: totalDocs}) } // Async re-index — extract paths before goroutine to allow arc GC paths := collectMemoryPaths(arc) go h.reindexMemoryPaths(context.WithoutCancel(ctx), ag.ID.String(), paths) return nil } func (h *AgentsHandler) importKG(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) error { if progressFn != nil { progressFn(ProgressEvent{Phase: "knowledge_graph", Status: "running", Total: len(arc.kgEntities)}) } if err := h.ingestKGByUser(ctx, ag.ID.String(), arc); err != nil { return fmt.Errorf("ingest kg: %w", err) } summary.KGEntities = len(arc.kgEntities) summary.KGRelations = len(arc.kgRelations) if progressFn != nil { progressFn(ProgressEvent{Phase: "knowledge_graph", Status: "done", Current: len(arc.kgEntities), Total: len(arc.kgEntities)}) } return nil } func (h *AgentsHandler) importCron(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) { tid := importTenantID(ctx) const paramsPerRow = 10 // agent_id, name, schedule_kind, cron_expression, interval_ms, run_at, timezone, payload, delete_after_run, tenant_id (enabled is literal false) const chunkSize = 5000 for start := 0; start < len(arc.cronJobs); start += chunkSize { end := min(start+chunkSize, len(arc.cronJobs)) chunk := arc.cronJobs[start:end] args := make([]any, 0, len(chunk)*paramsPerRow) placeholders := make([]string, 0, len(chunk)) for i, j := range chunk { base := i * paramsPerRow placeholders = append(placeholders, fmt.Sprintf( "($%d,$%d,false,$%d,$%d,$%d,$%d,$%d,$%d,$%d,$%d)", base+1, base+2, base+3, base+4, base+5, base+6, base+7, base+8, base+9, base+10, )) args = append(args, ag.ID, j.Name, j.ScheduleKind, j.CronExpression, j.IntervalMS, nullStr(j.RunAt), j.Timezone, j.Payload, j.DeleteAfterRun, tid, ) } q := `INSERT INTO cron_jobs (agent_id, name, enabled, schedule_kind, cron_expression, interval_ms, run_at, timezone, payload, delete_after_run, tenant_id) VALUES ` + strings.Join(placeholders, ",") + ` ON CONFLICT (agent_id, tenant_id, name) DO UPDATE SET schedule_kind = EXCLUDED.schedule_kind, cron_expression = EXCLUDED.cron_expression, interval_ms = EXCLUDED.interval_ms, run_at = EXCLUDED.run_at, timezone = EXCLUDED.timezone, payload = EXCLUDED.payload, delete_after_run = EXCLUDED.delete_after_run` if _, err := h.db.ExecContext(ctx, q, args...); err != nil { slog.Warn("agents.import.cron_jobs.batch", "agent_id", ag.ID, "count", len(chunk), "error", err) } summary.CronJobs += len(chunk) } if progressFn != nil { progressFn(ProgressEvent{Phase: "cron", Status: "done", Current: summary.CronJobs, Total: len(arc.cronJobs)}) } } func (h *AgentsHandler) importUserProfiles(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) { // workspace=NULL for portability (auto-created via GetOrCreateUserProfile on first user access) tid := importTenantID(ctx) const colsPerRow = 3 // agent_id, user_id, tenant_id const chunkSize = 5000 for start := 0; start < len(arc.userProfiles); start += chunkSize { end := min(start+chunkSize, len(arc.userProfiles)) chunk := arc.userProfiles[start:end] args := make([]any, 0, len(chunk)*colsPerRow) placeholders := make([]string, 0, len(chunk)) for i, p := range chunk { base := i * colsPerRow placeholders = append(placeholders, fmt.Sprintf("($%d,$%d,NULL,$%d)", base+1, base+2, base+3)) args = append(args, ag.ID, p.UserID, tid) } q := `INSERT INTO user_agent_profiles (agent_id, user_id, workspace, tenant_id) VALUES ` + strings.Join(placeholders, ",") + ` ON CONFLICT (agent_id, user_id) DO NOTHING` if _, err := h.db.ExecContext(ctx, q, args...); err != nil { slog.Warn("agents.import.user_profiles.batch", "agent_id", ag.ID, "count", len(chunk), "error", err) } summary.UserProfiles += len(chunk) } if progressFn != nil { progressFn(ProgressEvent{Phase: "user_profiles", Status: "done", Current: summary.UserProfiles, Total: len(arc.userProfiles)}) } } func (h *AgentsHandler) importUserOverrides(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) { tid := importTenantID(ctx) const colsPerRow = 6 // agent_id, user_id, provider, model, settings, tenant_id const chunkSize = 5000 for start := 0; start < len(arc.userOverrides); start += chunkSize { end := min(start+chunkSize, len(arc.userOverrides)) chunk := arc.userOverrides[start:end] args := make([]any, 0, len(chunk)*colsPerRow) placeholders := make([]string, 0, len(chunk)) for i, o := range chunk { base := i * colsPerRow placeholders = append(placeholders, fmt.Sprintf( "($%d,$%d,$%d,$%d,$%d,$%d)", base+1, base+2, base+3, base+4, base+5, base+6, )) args = append(args, ag.ID, o.UserID, o.Provider, o.Model, coalesceJSON(o.Settings), tid) } q := `INSERT INTO user_agent_overrides (agent_id, user_id, provider, model, settings, tenant_id) VALUES ` + strings.Join(placeholders, ",") + ` ON CONFLICT (agent_id, user_id) DO UPDATE SET provider = EXCLUDED.provider, model = EXCLUDED.model, settings = EXCLUDED.settings, updated_at = NOW()` if _, err := h.db.ExecContext(ctx, q, args...); err != nil { slog.Warn("agents.import.user_overrides.batch", "agent_id", ag.ID, "count", len(chunk), "error", err) } summary.UserOverrides += len(chunk) } if progressFn != nil { progressFn(ProgressEvent{Phase: "user_overrides", Status: "done", Current: summary.UserOverrides, Total: len(arc.userOverrides)}) } } func (h *AgentsHandler) importWorkspace(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) { wsPath := config.ExpandHome(fmt.Sprintf("%s/%s", h.defaultWorkspace, ag.AgentKey)) imported, wsErr := extractWorkspaceFiles(wsPath, arc.workspaceFiles, false) if wsErr != nil { slog.Warn("import: workspace extraction failed", "path", wsPath, "error", wsErr) } summary.WorkspaceFiles = imported if progressFn != nil { progressFn(ProgressEvent{Phase: "workspace", Status: "done", Current: imported, Total: len(arc.workspaceFiles)}) } } func (h *AgentsHandler) importEpisodic(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) { if progressFn != nil { progressFn(ProgressEvent{Phase: "episodic", Status: "running", Total: len(arc.episodicSummaries)}) } tid := importTenantID(ctx) for _, ep := range arc.episodicSummaries { exists, _ := h.episodicStore.ExistsBySourceID(ctx, ag.ID.String(), ep.UserID, ep.SourceID) if exists { continue } epSum := &store.EpisodicSummary{ TenantID: tid, AgentID: ag.ID, UserID: ep.UserID, SessionKey: ep.SessionKey, Summary: ep.Summary, KeyTopics: ep.KeyTopics, L0Abstract: ep.L0Abstract, SourceType: ep.SourceType, SourceID: ep.SourceID, TurnCount: ep.TurnCount, TokenCount: ep.TokenCount, // Embedding: nil (not exported; re-index separately) // ExpiresAt: not restored (summaries have no expiry on import) } if err := h.episodicStore.Create(ctx, epSum); err != nil { slog.Warn("agents.import.episodic", "agent_id", ag.ID, "source_id", ep.SourceID, "error", err) continue } summary.EpisodicSummaries++ } if progressFn != nil { progressFn(ProgressEvent{Phase: "episodic", Status: "done", Current: summary.EpisodicSummaries, Total: len(arc.episodicSummaries)}) } } func (h *AgentsHandler) importEvolution(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) { tid := importTenantID(ctx) // Metrics are time-series: re-import duplicates are acceptable for v1. if len(arc.evolutionMetrics) > 0 { if progressFn != nil { progressFn(ProgressEvent{Phase: "evolution_metrics", Status: "running", Total: len(arc.evolutionMetrics)}) } for _, m := range arc.evolutionMetrics { _, err := h.db.ExecContext(ctx, `INSERT INTO agent_evolution_metrics (agent_id, session_key, metric_type, metric_key, value, created_at, tenant_id) VALUES ($1, $2, $3, $4, $5, $6::timestamptz, $7)`, ag.ID, m.SessionKey, m.MetricType, m.MetricKey, nullJSON(m.Value), m.CreatedAt, tid, ) if err != nil { slog.Warn("agents.import.evolution_metric", "agent_id", ag.ID, "error", err) continue } summary.EvolutionMetrics++ } if progressFn != nil { progressFn(ProgressEvent{Phase: "evolution_metrics", Status: "done", Current: summary.EvolutionMetrics, Total: len(arc.evolutionMetrics)}) } } // Suggestions: dedup by (agent_id, suggestion_type, suggestion) via SELECT EXISTS. if len(arc.evolutionSuggestions) > 0 { if progressFn != nil { progressFn(ProgressEvent{Phase: "evolution_suggestions", Status: "running", Total: len(arc.evolutionSuggestions)}) } for _, s := range arc.evolutionSuggestions { var exists bool _ = h.db.QueryRowContext(ctx, `SELECT EXISTS(SELECT 1 FROM agent_evolution_suggestions WHERE agent_id = $1 AND suggestion_type = $2 AND suggestion = $3 AND tenant_id = $4)`, ag.ID, s.SuggestionType, s.Suggestion, tid, ).Scan(&exists) if exists { continue } _, err := h.db.ExecContext(ctx, `INSERT INTO agent_evolution_suggestions (agent_id, suggestion_type, suggestion, rationale, parameters, status, reviewed_by, reviewed_at, created_at, tenant_id) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::timestamptz, $9::timestamptz, $10)`, ag.ID, s.SuggestionType, s.Suggestion, s.Rationale, nullJSON(s.Parameters), s.Status, nullStrVal(s.ReviewedBy), nullStr(s.ReviewedAt), s.CreatedAt, tid, ) if err != nil { slog.Warn("agents.import.evolution_suggestion", "agent_id", ag.ID, "type", s.SuggestionType, "error", err) continue } summary.EvolutionSuggestions++ } if progressFn != nil { progressFn(ProgressEvent{Phase: "evolution_suggestions", Status: "done", Current: summary.EvolutionSuggestions, Total: len(arc.evolutionSuggestions)}) } } } func (h *AgentsHandler) importVault(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, progressFn func(ProgressEvent)) { tid := importTenantID(ctx) if progressFn != nil { progressFn(ProgressEvent{Phase: "vault_documents", Status: "running", Total: len(arc.vaultDocuments)}) } for _, d := range arc.vaultDocuments { agentIDStr := ag.ID.String() doc := &store.VaultDocument{ TenantID: tid.String(), AgentID: &agentIDStr, TeamID: nil, // team_id not portable Scope: d.Scope, CustomScope: d.CustomScope, Path: d.Path, Title: d.Title, DocType: d.DocType, ContentHash: d.ContentHash, Summary: d.Summary, // Embedding nil — re-indexed by vault FS sync } if len(d.Metadata) > 0 { if err := json.Unmarshal(d.Metadata, &doc.Metadata); err != nil { doc.Metadata = nil } } if err := h.vaultStore.UpsertDocument(ctx, doc); err != nil { slog.Warn("agents.import.vault_doc", "agent_id", ag.ID, "path", d.Path, "error", err) continue } summary.VaultDocuments++ } if progressFn != nil { progressFn(ProgressEvent{Phase: "vault_documents", Status: "done", Current: summary.VaultDocuments, Total: len(arc.vaultDocuments)}) } // Two-pass link import: build pathToID map first if len(arc.vaultLinks) > 0 { h.importVaultLinks(ctx, ag, arc, summary, tid, progressFn) } } func (h *AgentsHandler) importVaultLinks(ctx context.Context, ag *store.AgentData, arc *importArchive, summary *ImportSummary, tid uuid.UUID, progressFn func(ProgressEvent)) { if progressFn != nil { progressFn(ProgressEvent{Phase: "vault_links", Status: "running", Total: len(arc.vaultLinks)}) } pathToID := make(map[string]string) rows, qErr := h.db.QueryContext(ctx, `SELECT id, path FROM vault_documents WHERE agent_id = $1 AND tenant_id = $2`, ag.ID, tid, ) if qErr == nil { for rows.Next() { var id, path string if err := rows.Scan(&id, &path); err == nil { pathToID[path] = id } } if err := rows.Err(); err != nil { slog.Warn("agents.import.vault_link_resolution", "error", err) } rows.Close() } for _, l := range arc.vaultLinks { fromID, ok1 := pathToID[l.FromDocPath] toID, ok2 := pathToID[l.ToDocPath] if !ok1 || !ok2 { // Target doc not found — skip gracefully continue } link := &store.VaultLink{ FromDocID: fromID, ToDocID: toID, LinkType: l.LinkType, Context: l.Context, } if err := h.vaultStore.CreateLink(ctx, link); err != nil { slog.Warn("agents.import.vault_link", "agent_id", ag.ID, "from", l.FromDocPath, "to", l.ToDocPath, "error", err) continue } summary.VaultLinks++ } if progressFn != nil { progressFn(ProgressEvent{Phase: "vault_links", Status: "done", Current: summary.VaultLinks, Total: len(arc.vaultLinks)}) } }