mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-14 00:16:38 +00:00
fix(vault): sanitize context window to prevent invalid UTF-8 (#1536)
ExtractWikilinks builds a ±25-byte context window around each wikilink
match. When the content contains multi-byte UTF-8 characters (Vietnamese,
CJK, emoji), the byte-offset start/end can fall on a continuation byte,
producing invalid UTF-8 that PostgreSQL rejects with SQLSTATE 22021
('invalid byte sequence for encoding UTF8').
Fix: wrap the byte-offset slice with strings.ToValidUTF8, which strips
any leading/trailing partial runes. This is a minimal one-line change
that preserves the existing context semantics while ensuring the output
is always valid UTF-8.
Available since Go 1.21 (this project uses Go 1.26).
Closes #1472
This commit is contained in:
1 parent
fc8a35eebd
commit
be573f411b
2 files changed
+35
-2
No files matched your search
@@ -33,10 +33,14 @@ func ExtractWikilinks(content string) []WikilinkMatch {
|
||||
continue
|
||||
}
|
||||
|
||||
// Build context: ~25 chars before and after the link
|
||||
// Build context: ~25 bytes before and after the link.
|
||||
// Sanitize with ToValidUTF8 so the byte-offset window does not
|
||||
// split a multi-byte rune (Vietnamese, CJK, emoji), which would
|
||||
// produce invalid UTF-8 that PostgreSQL rejects with
|
||||
// "invalid byte sequence for encoding UTF8" (SQLSTATE 22021).
|
||||
start := max(m[0]-25, 0)
|
||||
end := min(m[1]+25, len(content))
|
||||
ctx := content[start:end]
|
||||
ctx := strings.ToValidUTF8(content[start:end], "")
|
||||
|
||||
result = append(result, WikilinkMatch{
|
||||
Target: target,
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// ============================================================================
|
||||
@@ -467,3 +469,30 @@ func hashRef(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Multi-byte UTF-8 Context Tests
|
||||
// ============================================================================
|
||||
|
||||
// TestExtractWikilinks_MultibyteUTF8Context reproduces issue #1472:
|
||||
// the byte-offset context window (±25 bytes) can split multi-byte UTF-8
|
||||
// characters (Vietnamese, CJK, emoji), producing invalid UTF-8 that
|
||||
// PostgreSQL rejects with "invalid byte sequence for encoding UTF8"
|
||||
// (SQLSTATE 22021).
|
||||
//
|
||||
// "ế" (U+1EBF) is 3 bytes in UTF-8. Ten of them = 30 bytes; adding a
|
||||
// single ASCII byte shifts the window start to byte 7, which lands on
|
||||
// a continuation byte of the 3rd "ế" — an invalid UTF-8 start.
|
||||
func TestExtractWikilinks_MultibyteUTF8Context(t *testing.T) {
|
||||
prefix := strings.Repeat("ế", 10) + "x" // 31 bytes
|
||||
suffix := "x" + strings.Repeat("ệ", 10) // 31 bytes
|
||||
content := prefix + " [[target]] " + suffix
|
||||
|
||||
matches := ExtractWikilinks(content)
|
||||
if len(matches) != 1 {
|
||||
t.Fatalf("ExtractWikilinks returned %d matches, want 1", len(matches))
|
||||
}
|
||||
if !utf8.ValidString(matches[0].Context) {
|
||||
t.Errorf("context contains invalid UTF-8: %q", matches[0].Context)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user