mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-13 08:15:29 +00:00
Duc Nguyen
016263eb84
bufio.Scanner has a fixed max-token size (SSEScanBufMax, 1MB). Codex's native image_generation streams a full base64 image in a SINGLE SSE data line (response.image_generation_call.partial_image and the final image_generation_call item), which regularly exceeds 1MB for anything larger than a thumbnail. Scanner.Scan() then fails with: stream read error: bufio.Scanner: token too long killing the whole turn after the model already generated the image. Replace bufio.Scanner with bufio.Reader.ReadString, which grows to fit a line of any length instead of erroring past a fixed cap. SSEScanBufMax is now unused and removed; SSEScanBufInit is reused as the reader's initial buffer size.
46 lines
1.3 KiB
Go
46 lines
1.3 KiB
Go
package providers
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// Reproduces the image-generation failure: a single SSE data line carrying a
|
|
// multi-MB base64 image used to overflow bufio.Scanner ("token too long").
|
|
func TestSSEScannerHandlesMultiMBLine(t *testing.T) {
|
|
big := strings.Repeat("A", 3*1024*1024) // 3MB, well over the old 1MB cap
|
|
stream := "event: response.image_generation_call.partial_image\n" +
|
|
"data: " + big + "\n" +
|
|
"data: [DONE]\n"
|
|
sc := NewSSEScanner(strings.NewReader(stream))
|
|
if !sc.Next() {
|
|
t.Fatalf("expected first data line; err=%v", sc.Err())
|
|
}
|
|
if len(sc.Data()) != len(big) {
|
|
t.Fatalf("data len=%d want %d", len(sc.Data()), len(big))
|
|
}
|
|
if sc.EventType() != "response.image_generation_call.partial_image" {
|
|
t.Fatalf("eventType=%q", sc.EventType())
|
|
}
|
|
if sc.Next() {
|
|
t.Fatalf("expected [DONE] to end stream, got %q", sc.Data())
|
|
}
|
|
if sc.Err() != nil {
|
|
t.Fatalf("unexpected err: %v", sc.Err())
|
|
}
|
|
}
|
|
|
|
// Final data line arriving with EOF and no trailing newline must be delivered.
|
|
func TestSSEScannerFinalLineNoNewline(t *testing.T) {
|
|
sc := NewSSEScanner(strings.NewReader("data: hello"))
|
|
if !sc.Next() || sc.Data() != "hello" {
|
|
t.Fatalf("data=%q err=%v", sc.Data(), sc.Err())
|
|
}
|
|
if sc.Next() {
|
|
t.Fatalf("expected end")
|
|
}
|
|
if sc.Err() != nil {
|
|
t.Fatalf("err=%v", sc.Err())
|
|
}
|
|
}
|