mirror of
https://github.com/tiennm99/goclaw.git
synced 2026-10-15 14:16:30 +00:00
fix(web_fetch): increase read limit and add empty content detection
- Read limit: maxChars*4 → max(maxChars*10, 512KB) to handle pages with large <head> sections (WordPress sites often have 30-50KB+ heads) - Add warning message when HTML extraction returns empty despite non-empty response body (bot protection, JS-only pages) - Enable HTTP/2 via ForceAttemptHTTP2 on custom Transport Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
dbd317ac81
commit
25be8cc41b
1 file changed
+8
-2
@@ -140,6 +140,7 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
|
||||
client := &http.Client{
|
||||
Timeout: time.Duration(fetchTimeoutSeconds) * time.Second,
|
||||
Transport: &http.Transport{
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
TLSHandshakeTimeout: 15 * time.Second,
|
||||
@@ -163,8 +164,9 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Limit body reading to avoid memory issues
|
||||
limitReader := io.LimitReader(resp.Body, int64(maxChars*4)) // read extra for HTML overhead
|
||||
// Read enough HTML to reach <body> content — pages often have 30-50KB+ <head> sections.
|
||||
readLimit := int64(max(maxChars*10, 512*1024))
|
||||
limitReader := io.LimitReader(resp.Body, readLimit)
|
||||
body, err := io.ReadAll(limitReader)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read body: %w", err)
|
||||
@@ -196,6 +198,10 @@ func (t *WebFetchTool) doFetch(ctx context.Context, rawURL, extractMode string,
|
||||
text = htmlToText(string(body))
|
||||
extractor = "html-to-text"
|
||||
}
|
||||
if text == "" && len(body) > 0 {
|
||||
text = "[No content extracted. The page may require JavaScript to render, " +
|
||||
"or returned a bot-protection challenge. Try using browser automation instead.]"
|
||||
}
|
||||
|
||||
default:
|
||||
text = string(body)
|
||||
|
||||
Reference in new issue
Block a user