fix(vault): sanitize context window to prevent invalid UTF-8 (#1536)

ExtractWikilinks builds a ±25-byte context window around each wikilink
match. When the content contains multi-byte UTF-8 characters (Vietnamese,
CJK, emoji), the byte-offset start/end can fall on a continuation byte,
producing invalid UTF-8 that PostgreSQL rejects with SQLSTATE 22021
('invalid byte sequence for encoding UTF8').

Fix: wrap the byte-offset slice with strings.ToValidUTF8, which strips
any leading/trailing partial runes. This is a minimal one-line change
that preserves the existing context semantics while ensuring the output
is always valid UTF-8.

Available since Go 1.21 (this project uses Go 1.26).

Closes #1472
This commit is contained in:
fchengyan authored and GitHub committed 2026-08-27 19:12:48 +07:00
1 parent fc8a35eebd
commit be573f411b
2 files changed
+35 -2

No files matched your search

+6 -2
View File
@@ -33,10 +33,14 @@ func ExtractWikilinks(content string) []WikilinkMatch {
continue
}
// Build context: ~25 chars before and after the link
// Build context: ~25 bytes before and after the link.
// Sanitize with ToValidUTF8 so the byte-offset window does not
// split a multi-byte rune (Vietnamese, CJK, emoji), which would
// produce invalid UTF-8 that PostgreSQL rejects with
// "invalid byte sequence for encoding UTF8" (SQLSTATE 22021).
start := max(m[0]-25, 0)
end := min(m[1]+25, len(content))
ctx := content[start:end]
ctx := strings.ToValidUTF8(content[start:end], "")
result = append(result, WikilinkMatch{
Target: target,
+29
View File
@@ -5,7 +5,9 @@ import (
"crypto/sha256"
"encoding/hex"
"os"
"strings"
"testing"
"unicode/utf8"
)
// ============================================================================
@@ -467,3 +469,30 @@ func hashRef(data []byte) string {
h := sha256.Sum256(data)
return hex.EncodeToString(h[:])
}
// ============================================================================
// Multi-byte UTF-8 Context Tests
// ============================================================================
// TestExtractWikilinks_MultibyteUTF8Context reproduces issue #1472:
// the byte-offset context window (±25 bytes) can split multi-byte UTF-8
// characters (Vietnamese, CJK, emoji), producing invalid UTF-8 that
// PostgreSQL rejects with "invalid byte sequence for encoding UTF8"
// (SQLSTATE 22021).
//
// "ế" (U+1EBF) is 3 bytes in UTF-8. Ten of them = 30 bytes; adding a
// single ASCII byte shifts the window start to byte 7, which lands on
// a continuation byte of the 3rd "ế" — an invalid UTF-8 start.
func TestExtractWikilinks_MultibyteUTF8Context(t *testing.T) {
prefix := strings.Repeat("ế", 10) + "x" // 31 bytes
suffix := "x" + strings.Repeat("ệ", 10) // 31 bytes
content := prefix + " [[target]] " + suffix
matches := ExtractWikilinks(content)
if len(matches) != 1 {
t.Fatalf("ExtractWikilinks returned %d matches, want 1", len(matches))
}
if !utf8.ValidString(matches[0].Context) {
t.Errorf("context contains invalid UTF-8: %q", matches[0].Context)
}
}