diff --git a/internal/vault/links.go b/internal/vault/links.go index ce14c62a..10bc9794 100644 --- a/internal/vault/links.go +++ b/internal/vault/links.go @@ -33,10 +33,14 @@ func ExtractWikilinks(content string) []WikilinkMatch { continue } - // Build context: ~25 chars before and after the link + // Build context: ~25 bytes before and after the link. + // Sanitize with ToValidUTF8 so the byte-offset window does not + // split a multi-byte rune (Vietnamese, CJK, emoji), which would + // produce invalid UTF-8 that PostgreSQL rejects with + // "invalid byte sequence for encoding UTF8" (SQLSTATE 22021). start := max(m[0]-25, 0) end := min(m[1]+25, len(content)) - ctx := content[start:end] + ctx := strings.ToValidUTF8(content[start:end], "") result = append(result, WikilinkMatch{ Target: target, diff --git a/internal/vault/vault_test.go b/internal/vault/vault_test.go index 5febb52b..8946277f 100644 --- a/internal/vault/vault_test.go +++ b/internal/vault/vault_test.go @@ -5,7 +5,9 @@ import ( "crypto/sha256" "encoding/hex" "os" + "strings" "testing" + "unicode/utf8" ) // ============================================================================ @@ -467,3 +469,30 @@ func hashRef(data []byte) string { h := sha256.Sum256(data) return hex.EncodeToString(h[:]) } + +// ============================================================================ +// Multi-byte UTF-8 Context Tests +// ============================================================================ + +// TestExtractWikilinks_MultibyteUTF8Context reproduces issue #1472: +// the byte-offset context window (±25 bytes) can split multi-byte UTF-8 +// characters (Vietnamese, CJK, emoji), producing invalid UTF-8 that +// PostgreSQL rejects with "invalid byte sequence for encoding UTF8" +// (SQLSTATE 22021). +// +// "ế" (U+1EBF) is 3 bytes in UTF-8. Ten of them = 30 bytes; adding a +// single ASCII byte shifts the window start to byte 7, which lands on +// a continuation byte of the 3rd "ế" — an invalid UTF-8 start. +func TestExtractWikilinks_MultibyteUTF8Context(t *testing.T) { + prefix := strings.Repeat("ế", 10) + "x" // 31 bytes + suffix := "x" + strings.Repeat("ệ", 10) // 31 bytes + content := prefix + " [[target]] " + suffix + + matches := ExtractWikilinks(content) + if len(matches) != 1 { + t.Fatalf("ExtractWikilinks returned %d matches, want 1", len(matches)) + } + if !utf8.ValidString(matches[0].Context) { + t.Errorf("context contains invalid UTF-8: %q", matches[0].Context) + } +}