mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-13 00:14:33 +00:00
fix(vault): centralize enrich skip filter and fix stop cancel bug
Extract shouldSkipEnrichment() replacing 4 scattered goclaw_gen_ checks. Filter also skips UUID, hex hash, digit-only, short, and known junk filenames. Fix cancelFuncs never being populated — Stop() was a no-op leaving UI stuck.
This commit is contained in:
1 parent
70697988dd
commit
1ea24ef8e2
4 files changed
+74
-17
No files matched your search
@@ -6,7 +6,6 @@ import (
|
||||
"log/slog"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/nextlevelbuilder/goclaw/internal/providers"
|
||||
"github.com/nextlevelbuilder/goclaw/internal/store"
|
||||
@@ -154,8 +153,8 @@ func (w *EnrichWorker) gatherCandidates(ctx context.Context, tenantID, _ string,
|
||||
if n.Score < enrichSimilarityMin || n.Document.Summary == "" {
|
||||
continue
|
||||
}
|
||||
// Skip auto-generated media files as link targets — they create noise.
|
||||
if strings.HasPrefix(n.Document.PathBasename, "goclaw_gen_") {
|
||||
// Skip meaningless filenames as link targets — they create noise.
|
||||
if shouldSkipEnrichment(n.Document.PathBasename) {
|
||||
continue
|
||||
}
|
||||
// Bidirectional dedup: only process each pair once.
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package vault
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Compiled patterns for meaningless filenames that should skip enrichment.
|
||||
var (
|
||||
reDigitsOnly = regexp.MustCompile(`^[0-9]+$`)
|
||||
reUUID = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
|
||||
reHexHash = regexp.MustCompile(`(?i)^[0-9a-f]{8,}$`)
|
||||
reMixedJunk = regexp.MustCompile(`(?i)^(img|tmp|temp|dsc|screenshot|untitled|file|pic|photo|vid|clip|scan|page)[_-][0-9]+$`)
|
||||
)
|
||||
|
||||
// shouldSkipEnrichment returns true if the basename indicates a file that
|
||||
// would produce noise during enrichment (auto-generated, hash-named, etc.).
|
||||
// Checks are applied to the stem (basename without extension).
|
||||
// Unicode/CJK filenames pass through — they carry semantic meaning.
|
||||
func shouldSkipEnrichment(basename string) bool {
|
||||
// Strip extension to get stem.
|
||||
stem := basename
|
||||
if idx := strings.LastIndex(basename, "."); idx > 0 {
|
||||
stem = basename[:idx]
|
||||
}
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(stem, "goclaw_gen_"):
|
||||
slog.Debug("vault.enrich: skip_generated", "file", basename)
|
||||
return true
|
||||
case len(stem) < 3:
|
||||
slog.Debug("vault.enrich: skip_short", "file", basename)
|
||||
return true
|
||||
case reDigitsOnly.MatchString(stem):
|
||||
slog.Debug("vault.enrich: skip_digits", "file", basename)
|
||||
return true
|
||||
case reUUID.MatchString(stem):
|
||||
slog.Debug("vault.enrich: skip_uuid", "file", basename)
|
||||
return true
|
||||
case reHexHash.MatchString(stem):
|
||||
slog.Debug("vault.enrich: skip_hex_hash", "file", basename)
|
||||
return true
|
||||
case reMixedJunk.MatchString(stem):
|
||||
slog.Debug("vault.enrich: skip_mixed_junk", "file", basename)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -99,18 +99,20 @@ func (w *EnrichWorker) resolveProviderForTenant(ctx context.Context, tenantID st
|
||||
// Stop cancels in-flight enrichment for the given tenant.
|
||||
// Safe to call even if no enrichment is running.
|
||||
func (w *EnrichWorker) Stop(tenantID string) {
|
||||
if cancel, ok := w.cancelFuncs.Load(tenantID); ok {
|
||||
if cancel, ok := w.cancelFuncs.LoadAndDelete(tenantID); ok {
|
||||
cancel.(context.CancelFunc)()
|
||||
w.cancelFuncs.Delete(tenantID)
|
||||
w.progress.Finish()
|
||||
slog.Info("vault.enrich: stopped by user", "tenant", tenantID)
|
||||
}
|
||||
// Always finish progress — ensures UI resets even if cancelFuncs was empty.
|
||||
w.progress.Finish()
|
||||
slog.Info("vault.enrich: stopped by user", "tenant", tenantID)
|
||||
}
|
||||
|
||||
// IsRunning returns true if enrichment is in progress for the tenant.
|
||||
func (w *EnrichWorker) IsRunning(tenantID string) bool {
|
||||
_, ok := w.cancelFuncs.Load(tenantID)
|
||||
return ok
|
||||
if _, ok := w.cancelFuncs.Load(tenantID); ok {
|
||||
return true
|
||||
}
|
||||
return w.progress.Status().Running
|
||||
}
|
||||
|
||||
// EnqueueUnenriched fetches documents with empty summary and emits enrichment events.
|
||||
@@ -127,8 +129,8 @@ func (w *EnrichWorker) EnqueueUnenriched(ctx context.Context, tenantID, workspac
|
||||
|
||||
count := 0
|
||||
for _, doc := range docs {
|
||||
// Skip auto-generated media files — they create noise links.
|
||||
if strings.HasPrefix(filepath.Base(doc.Path), "goclaw_gen_") {
|
||||
// Skip meaningless filenames — they create noise links.
|
||||
if shouldSkipEnrichment(filepath.Base(doc.Path)) {
|
||||
continue
|
||||
}
|
||||
agentID := ""
|
||||
@@ -178,10 +180,8 @@ func (w *EnrichWorker) Handle(ctx context.Context, event eventbus.DomainEvent) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// Skip auto-generated media files (goclaw_gen_*) — they create excessive
|
||||
// noise links due to similar embeddings. These are typically image outputs
|
||||
// that don't benefit from semantic linking.
|
||||
if basename := filepath.Base(payload.Path); strings.HasPrefix(basename, "goclaw_gen_") {
|
||||
// Skip meaningless filenames — they create noise links.
|
||||
if shouldSkipEnrichment(filepath.Base(payload.Path)) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -201,7 +201,12 @@ func (w *EnrichWorker) Handle(ctx context.Context, event eventbus.DomainEvent) e
|
||||
return nil // another goroutine already processing this agent's queue
|
||||
}
|
||||
|
||||
w.processBatch(ctx, key)
|
||||
// Create per-tenant cancel context for stop capability.
|
||||
cancelCtx, cancel := context.WithCancel(ctx)
|
||||
w.cancelFuncs.Store(payload.TenantID, cancel)
|
||||
w.processBatch(cancelCtx, key)
|
||||
// Clean up after batch completes naturally.
|
||||
w.cancelFuncs.Delete(payload.TenantID)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -217,6 +222,10 @@ type enriched struct {
|
||||
// overwhelm the LLM provider with hundreds of concurrent requests.
|
||||
func (w *EnrichWorker) processBatch(ctx context.Context, key string) {
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
w.queue.TryFinish(key)
|
||||
return
|
||||
}
|
||||
items := w.queue.Drain(key)
|
||||
if len(items) == 0 {
|
||||
if w.queue.TryFinish(key) {
|
||||
|
||||
@@ -117,7 +117,7 @@ func RescanWorkspace(ctx context.Context, params RescanParams, vs store.VaultSto
|
||||
// can call progress.Start(total) before workers receive events.
|
||||
// Skip auto-generated media files (goclaw_gen_*) — they create excessive
|
||||
// noise links and shouldn't be counted in progress tracking.
|
||||
if bus != nil && !strings.HasPrefix(filepath.Base(relPath), "goclaw_gen_") {
|
||||
if bus != nil && !shouldSkipEnrichment(filepath.Base(relPath)) {
|
||||
result.PendingEvents = append(result.PendingEvents, eventbus.DomainEvent{
|
||||
ID: uuid.Must(uuid.NewV7()).String(),
|
||||
Type: eventbus.EventVaultDocUpserted,
|
||||
|
||||
Reference in new issue
Block a user