From be573f411b2e9aca86ba3591f8b3078b0d646771 Mon Sep 17 00:00:00 2001
From: fchengyan <3181088318@qq.com>
Date: Thu, 27 Aug 2026 20:12:48 +0800
Subject: [PATCH] fix(vault): sanitize context window to prevent invalid UTF-8
 (#1536)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ExtractWikilinks builds a ±25-byte context window around each wikilink
match. When the content contains multi-byte UTF-8 characters (Vietnamese,
CJK, emoji), the byte-offset start/end can fall on a continuation byte,
producing invalid UTF-8 that PostgreSQL rejects with SQLSTATE 22021
('invalid byte sequence for encoding UTF8').

Fix: wrap the byte-offset slice with strings.ToValidUTF8, which strips
any leading/trailing partial runes. This is a minimal one-line change
that preserves the existing context semantics while ensuring the output
is always valid UTF-8.

Available since Go 1.21 (this project uses Go 1.26).

Closes #1472
---
 internal/vault/links.go      |  8 ++++++--
 internal/vault/vault_test.go | 29 +++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/internal/vault/links.go b/internal/vault/links.go
index ce14c62a..10bc9794 100644
--- a/internal/vault/links.go
+++ b/internal/vault/links.go
@@ -33,10 +33,14 @@ func ExtractWikilinks(content string) []WikilinkMatch {
 			continue
 		}
 
-		// Build context: ~25 chars before and after the link
+		// Build context: ~25 bytes before and after the link.
+		// Sanitize with ToValidUTF8 so the byte-offset window does not
+		// split a multi-byte rune (Vietnamese, CJK, emoji), which would
+		// produce invalid UTF-8 that PostgreSQL rejects with
+		// "invalid byte sequence for encoding UTF8" (SQLSTATE 22021).
 		start := max(m[0]-25, 0)
 		end := min(m[1]+25, len(content))
-		ctx := content[start:end]
+		ctx := strings.ToValidUTF8(content[start:end], "")
 
 		result = append(result, WikilinkMatch{
 			Target:  target,
diff --git a/internal/vault/vault_test.go b/internal/vault/vault_test.go
index 5febb52b..8946277f 100644
--- a/internal/vault/vault_test.go
+++ b/internal/vault/vault_test.go
@@ -5,7 +5,9 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"os"
+	"strings"
 	"testing"
+	"unicode/utf8"
 )
 
 // ============================================================================
@@ -467,3 +469,30 @@ func hashRef(data []byte) string {
 	h := sha256.Sum256(data)
 	return hex.EncodeToString(h[:])
 }
+
+// ============================================================================
+// Multi-byte UTF-8 Context Tests
+// ============================================================================
+
+// TestExtractWikilinks_MultibyteUTF8Context reproduces issue #1472:
+// the byte-offset context window (±25 bytes) can split multi-byte UTF-8
+// characters (Vietnamese, CJK, emoji), producing invalid UTF-8 that
+// PostgreSQL rejects with "invalid byte sequence for encoding UTF8"
+// (SQLSTATE 22021).
+//
+// "ế" (U+1EBF) is 3 bytes in UTF-8. Ten of them = 30 bytes; adding a
+// single ASCII byte shifts the window start to byte 7, which lands on
+// a continuation byte of the 3rd "ế" — an invalid UTF-8 start.
+func TestExtractWikilinks_MultibyteUTF8Context(t *testing.T) {
+	prefix := strings.Repeat("ế", 10) + "x" // 31 bytes
+	suffix := "x" + strings.Repeat("ệ", 10)  // 31 bytes
+	content := prefix + " [[target]] " + suffix
+
+	matches := ExtractWikilinks(content)
+	if len(matches) != 1 {
+		t.Fatalf("ExtractWikilinks returned %d matches, want 1", len(matches))
+	}
+	if !utf8.ValidString(matches[0].Context) {
+		t.Errorf("context contains invalid UTF-8: %q", matches[0].Context)
+	}
+}