goclaw/internal/vault/rescan.go

package vault

import (
	"context"
	"log/slog"
	"path/filepath"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/nextlevelbuilder/goclaw/internal/eventbus"
	"github.com/nextlevelbuilder/goclaw/internal/store"
)

// RescanParams holds input for tenant-wide workspace rescan.
// AgentMap and TeamSet are pre-loaded by the caller to avoid per-file DB lookups.
type RescanParams struct {
	TenantID  string
	Workspace string            // absolute path to tenant's workspace root
	AgentMap  map[string]string // agent_key → agent UUID
	TeamSet   map[string]bool   // team UUID → exists (for validation)
}

// RescanResult holds the outcome of a workspace rescan.
type RescanResult struct {
	Scanned    int  `json:"scanned"`
	New        int  `json:"new"`
	Updated    int  `json:"updated"`
	Unchanged  int  `json:"unchanged"`
	Reenqueued int  `json:"reenqueued"` // docs re-enqueued for failed enrichment retry
	Skipped    int  `json:"skipped"`
	Errors     int  `json:"errors"`
	Truncated  bool `json:"truncated"`

	// PendingEvents holds enrichment events collected during scan.
	// Caller must publish these AFTER calling progress.Start(total)
	// to avoid race between event workers and progress tracking.
	PendingEvents []eventbus.DomainEvent `json:"-"`
}

// RescanWorkspace walks the tenant workspace and registers missing or changed
// files in vault_documents. Ownership (agent/team/scope) is inferred from path.
// Publishes EventVaultDocUpserted for each new or updated file so the
// enrichment worker can process them asynchronously.
func RescanWorkspace(ctx context.Context, params RescanParams, vs store.VaultStore, bus eventbus.DomainEventBus) (*RescanResult, error) {
	entries, walkStats, err := SafeWalkWorkspace(ctx, params.Workspace, DefaultWalkOptions())
	if err != nil {
		return nil, err
	}

	result := &RescanResult{
		Scanned:   walkStats.Eligible,
		Skipped:   walkStats.SkippedExcluded + walkStats.SkippedSymlinks + walkStats.SkippedTooLarge,
		Truncated: walkStats.Truncated,
	}

	for _, entry := range entries {
		agentID, teamID, chatID, scope, strippedPath := inferOwnerFromPath(entry.RelPath, params.AgentMap, params.TeamSet)
		if scope == "" {
			// Unknown agent key or invalid team UUID — skip.
			result.Skipped++
			continue
		}

		hash, hashErr := ContentHashFile(entry.AbsPath)
		if hashErr != nil {
			result.Errors++
			continue
		}

		// Resolve the agent ID string for store lookup (empty string = no agent filter).
		agentIDStr := ""
		if agentID != nil {
			agentIDStr = *agentID
		}

		// Use full workspace-relative path for DB storage and enrichment.
		// This preserves the real filesystem path (e.g. "teams/{uuid}/system/file.md")
		// so enrichment worker can locate files via filepath.Join(workspace, path).
		relPath := entry.RelPath

		// Check if document already exists with same hash.
		// Fallback: also check old stripped path (pre-fix records stored without prefix).
		existing, _ := vs.GetDocument(ctx, params.TenantID, agentIDStr, relPath)
		if existing == nil && strippedPath != relPath {
			existing, _ = vs.GetDocument(ctx, params.TenantID, agentIDStr, strippedPath)
		}
		if existing != nil && existing.ContentHash == hash {
			result.Unchanged++
			continue
		}

		doc := &store.VaultDocument{
			TenantID:    params.TenantID,
			AgentID:     agentID,
			TeamID:      teamID,
			ChatID:      chatID,
			Scope:       scope,
			Path:        relPath,
			Title:       InferTitle(relPath),
			DocType:     InferDocType(relPath),
			ContentHash: hash,
		}

		if err := vs.UpsertDocument(ctx, doc); err != nil {
			slog.Warn("vault.rescan: upsert", "path", relPath, "err", err)
			result.Errors++
			continue
		}

		if existing != nil {
			result.Updated++
		} else {
			result.New++
		}

		// Collect enrichment events — published AFTER the loop so the caller
		// can call progress.Start(total) before workers receive events.
		// Skip auto-generated media files (goclaw_gen_*) — they create excessive
		// noise links and shouldn't be counted in progress tracking.
		if bus != nil && !shouldSkipEnrichment(filepath.Base(relPath)) {
			result.PendingEvents = append(result.PendingEvents, eventbus.DomainEvent{
				ID:        uuid.Must(uuid.NewV7()).String(),
				Type:      eventbus.EventVaultDocUpserted,
				SourceID:  doc.ID + ":" + hash,
				TenantID:  params.TenantID,
				AgentID:   agentIDStr,
				Timestamp: time.Now(),
				Payload: eventbus.VaultDocUpsertedPayload{
					DocID:       doc.ID,
					TenantID:    params.TenantID,
					AgentID:     agentIDStr,
					Path:        relPath,
					ContentHash: hash,
					Workspace:   params.Workspace,
				},
			})
		}
	}

	slog.Info("vault.rescan",
		"tenant", params.TenantID,
		"scanned", result.Scanned, "new", result.New,
		"updated", result.Updated, "unchanged", result.Unchanged,
		"skipped", result.Skipped, "errors", result.Errors,
		"truncated", result.Truncated)

	return result, nil
}

// inferOwnerFromPath parses a tenant-relative path to determine ownership.
// Returns: agentID (*string), teamID (*string), chatID (*string), scope, strippedPath.
//
// Path patterns (checked in order):
//
//	teams/{team_uuid}/{chat}/...     → teamID=uuid, chatID=chat, scope="team"
//	teams/{team_uuid}/file.md        → teamID=uuid, chatID=nil (team-wide), scope="team"
//	agents/{agent_key}/...           → agentID=lookup(key), scope="personal"  (legacy prefix)
//	{agent_key}/...                  → agentID=lookup(key), scope="personal"  (workspace layout)
//
// Chat segments starting with "." (e.g. ".goclaw") are config dirs, not real chats — chatID stays nil.
// The full relPath is preserved in strippedPath for DB storage so enrichment workers
// can locate files via filepath.Join(workspace, path).
// Returns scope="" to signal the file should be skipped (unknown agent or invalid team).
func inferOwnerFromPath(relPath string, agentMap map[string]string, teamSet map[string]bool) (agentID *string, teamID *string, chatID *string, scope string, strippedPath string) {
	// Team paths: teams/{uuid}/[chat/]...
	if strings.HasPrefix(relPath, "teams/") {
		rest := relPath[len("teams/"):]
		id, remainder, hasSlash := strings.Cut(rest, "/")
		if !hasSlash || id == "" || strings.Contains(remainder, "..") {
			return nil, nil, nil, "", relPath
		}
		if _, parseErr := uuid.Parse(id); parseErr != nil {
			return nil, nil, nil, "", relPath
		}
		if !teamSet[id] {
			return nil, nil, nil, "", relPath
		}
		// Extract chat segment (second path component after team uuid) if present
		// and not a config/hidden dir. Paths without a chat segment stay team-wide.
		if chatSeg, _, hasChat := strings.Cut(remainder, "/"); hasChat && chatSeg != "" && !strings.HasPrefix(chatSeg, ".") {
			cid := chatSeg
			return nil, &id, &cid, "team", relPath
		}
		return nil, &id, nil, "team", relPath
	}

	// Agent paths: agents/{key}/... (legacy prefix) or {key}/... (actual workspace layout)
	if strings.HasPrefix(relPath, "agents/") {
		rest := relPath[len("agents/"):]
		key, _, hasSlash := strings.Cut(rest, "/")
		if hasSlash && key != "" && !strings.Contains(relPath, "..") {
			if agentUUID, ok := agentMap[key]; ok {
				return &agentUUID, nil, nil, "personal", relPath
			}
		}
		return nil, nil, nil, "", relPath
	}

	// Root-level agent_key match: {agent_key}/...
	// Workspace resolver uses agent_key as folder name directly.
	firstSeg, _, hasSlash := strings.Cut(relPath, "/")
	if hasSlash && firstSeg != "" {
		if agentUUID, ok := agentMap[firstSeg]; ok {
			return &agentUUID, nil, nil, "personal", relPath
		}
	}

	// Everything else is shared (root-level files, unknown folders)
	return nil, nil, nil, "shared", relPath
}

// InferDocType guesses doc_type from path conventions.
// Exported so both rescan and vault interceptor share the same logic.
//
// Path-prefix rules (memory/, skills/, episodic/, SOUL/IDENTITY/AGENTS)
// take precedence over extension classification — e.g. `memory/foo.md`
// classifies as `memory` even though `.md` is a whitelisted note extension.
// Phase 01: extension fallback uses the shared `extensionDocType` whitelist
// so PDFs/office files resolve to `document` instead of the prior `note`.
func InferDocType(relPath string) string {
	lower := strings.ToLower(relPath)
	ext := strings.ToLower(filepath.Ext(relPath))

	// Path-based overrides first (memory/ beats .md → note).
	switch {
	case strings.HasPrefix(lower, "memory/"):
		return "memory"
	case strings.Contains(lower, "soul.md") || strings.Contains(lower, "identity.md") || strings.Contains(lower, "agents.md"):
		return "context"
	case strings.HasPrefix(lower, "skills/") || strings.HasSuffix(lower, "skill.md"):
		return "skill"
	case strings.HasPrefix(lower, "episodic/"):
		return "episodic"
	}

	// Extension whitelist fallback — single source of truth shared with safe_walk.
	if _, dt := isIncludedExtension(ext); dt != "" {
		return dt
	}
	return "note"
}

// InferTitle extracts a human-readable title from a file path.
// Exported so both rescan and vault interceptor share the same logic.
func InferTitle(relPath string) string {
	base := filepath.Base(relPath)
	return strings.TrimSuffix(base, filepath.Ext(base))
}