feat(tools): local-first document extraction for read_document

Squash merge PR #111 after resolving docs/changelog conflicts. Local checks covered tools/config and both Go builds; PR CI run 26704622503 passed release-versioning, go, and web.

internal/tools/document_parser.go

@@ -0,0 +1,222 @@
+package tools
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Sentinel outcomes. Callers fall back to the cloud vision LLM chain on any of
+// these — they distinguish "this parser cannot/should-not handle the document"
+// from a genuine extraction success.
+var (
+	ErrParserUnsupported = errors.New("document_parser: unsupported mime")
+	ErrParserUnavailable = errors.New("document_parser: extractor binary not installed")
+	ErrParserEmpty       = errors.New("document_parser: extraction produced no usable text")
+)
+
+// MIME types the local extractor recognizes.
+const (
+	mimePDF  = "application/pdf"
+	mimeDOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+)
+
+// docTruncationMarker matches the marker read_document.go appends so truncated
+// local output reads identically to truncated direct-text output downstream.
+const docTruncationMarker = "\n\n[... truncated at 500KB ...]"
+
+// Default config values applied when a field is left zero.
+const (
+	defaultMaxPages   = 200
+	defaultTimeout    = 30 * time.Second
+	defaultMinTextLen = 16
+	// killGrace is how long a process group has to exit after SIGTERM before
+	// it is escalated to SIGKILL.
+	killGrace = 3 * time.Second
+	// stderrCap bounds captured stderr so a chatty extractor cannot block on a
+	// full pipe; only used for error messages.
+	stderrCap = 8 * 1024
+)
+
+// DocumentParser extracts plain text from a document file. Implementations MUST
+// be safe on untrusted input and MUST NOT execute embedded content. A future
+// LiteParse backend can satisfy this same interface without touching callers.
+type DocumentParser interface {
+	// Extract returns plain text, or a sentinel/wrapped error signalling the
+	// caller to fall back to the cloud vision path. mime selects the tool.
+	Extract(ctx context.Context, path, mime string) (string, error)
+	// Supports reports whether this parser handles the mime AND its binary is
+	// currently available (LookPath succeeds at call time).
+	Supports(mime string) bool
+}
+
+// LocalExtractConfig tunes LocalExtractParser. Zero values get sane defaults.
+type LocalExtractConfig struct {
+	Enabled    bool          // local-first on/off (default OFF — opt-in)
+	MaxPages   int           // 0 => 200; passed to pdftotext -l
+	Timeout    time.Duration // per-extraction timeout (0 => 30s)
+	MinTextLen int           // below this (after trim) => ErrParserEmpty (0 => 16)
+}
+
+// LocalExtractParser extracts text from PDF (pdftotext) and DOCX (pandoc) via
+// no-shell subprocess exec, with binary detection, process-group timeout kill,
+// a minimal environment, and output limits.
+type LocalExtractParser struct {
+	cfg LocalExtractConfig
+	// lookPath is injectable so tests can stub binary discovery without a
+	// PATH-prepend race. Defaults to exec.LookPath. Re-resolved per call so a
+	// binary installed at runtime is picked up without a restart.
+	lookPath func(string) (string, error)
+}
+
+// NewLocalExtractParser builds a parser, filling zero-valued config fields with
+// defaults. Enabled is taken as given — the construction site owns the toggle.
+func NewLocalExtractParser(cfg LocalExtractConfig) *LocalExtractParser {
+	if cfg.MaxPages <= 0 {
+		cfg.MaxPages = defaultMaxPages
+	}
+	if cfg.Timeout <= 0 {
+		cfg.Timeout = defaultTimeout
+	}
+	if cfg.MinTextLen <= 0 {
+		cfg.MinTextLen = defaultMinTextLen
+	}
+	return &LocalExtractParser{cfg: cfg, lookPath: exec.LookPath}
+}
+
+// commandFor maps a mime to its extractor binary name and arguments.
+//
+// pdftotext takes NO "--" option terminator: poppler's argument parser does not
+// treat "--" as one and would consume it as the input-file positional, pushing
+// the real path into the output-file slot (silently overwriting the source).
+// The path is the sole positional; the trailing "-" routes text to stdout.
+//
+// pandoc runs with --sandbox so an untrusted DOCX cannot make pandoc fetch
+// remote resources or read arbitrary local files during conversion.
+func (p *LocalExtractParser) commandFor(mime, path string) (bin string, args []string, err error) {
+	switch mime {
+	case mimePDF:
+		return "pdftotext", []string{"-q", "-enc", "UTF-8", "-l", strconv.Itoa(p.cfg.MaxPages), path, "-"}, nil
+	case mimeDOCX:
+		return "pandoc", []string{"--sandbox", "-t", "plain", "--wrap=none", path}, nil
+	default:
+		return "", nil, ErrParserUnsupported
+	}
+}
+
+// Supports reports whether the mime is handled and its binary is resolvable now.
+func (p *LocalExtractParser) Supports(mime string) bool {
+	if !p.cfg.Enabled {
+		return false
+	}
+	bin, _, err := p.commandFor(mime, "")
+	if err != nil {
+		return false
+	}
+	_, err = p.lookPath(bin)
+	return err == nil
+}
+
+// Extract runs the extractor for mime against path and returns plain text.
+// A timeout or non-zero exit always yields an error (never partial text) so the
+// caller falls back rather than treating a half-extracted document as complete.
+func (p *LocalExtractParser) Extract(ctx context.Context, path, mime string) (string, error) {
+	if !p.cfg.Enabled {
+		return "", ErrParserUnsupported
+	}
+	bin, args, err := p.commandFor(mime, path)
+	if err != nil {
+		return "", err
+	}
+	binPath, err := p.lookPath(bin)
+	if err != nil {
+		return "", fmt.Errorf("%w: %s", ErrParserUnavailable, bin)
+	}
+
+	cmd := exec.Command(binPath, args...) // NOT CommandContext — see kill() below
+	setProcessGroup(cmd)
+	// Minimal allowlist environment (PATH/HOME/LANG/USER); never inherit the
+	// gateway process env, which holds DB DSNs, API keys, and encryption keys.
+	cmd.Env = buildCredentialedEnv(nil)
+
+	stdout := &cappedWriter{cap: documentMaxTextBytes}
+	stderr := &cappedWriter{cap: stderrCap}
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+
+	if err := cmd.Start(); err != nil {
+		return "", fmt.Errorf("document_parser: start %s: %w", bin, err)
+	}
+
+	done := make(chan error, 1)
+	go func() { done <- cmd.Wait() }()
+
+	timer := time.NewTimer(p.cfg.Timeout)
+	defer timer.Stop()
+
+	select {
+	case werr := <-done:
+		if werr != nil {
+			return "", fmt.Errorf("document_parser: %s failed: %w (stderr: %s)",
+				bin, werr, strings.TrimSpace(stderr.buf.String()))
+		}
+		// Clean exit 0 — the only path that may return text.
+	case <-ctx.Done():
+		p.kill(cmd, done)
+		return "", fmt.Errorf("document_parser: %s cancelled: %w", bin, ctx.Err())
+	case <-timer.C:
+		p.kill(cmd, done)
+		return "", fmt.Errorf("document_parser: %s timed out after %s", bin, p.cfg.Timeout)
+	}
+
+	text := stdout.buf.String()
+	if len(strings.TrimSpace(text)) < p.cfg.MinTextLen {
+		// Scanned/image-only PDFs yield whitespace; route to vision fallback.
+		return "", ErrParserEmpty
+	}
+	if stdout.overflow {
+		text += docTruncationMarker
+	}
+	return text, nil
+}
+
+// kill terminates the whole process group: SIGTERM, a grace period, then
+// SIGKILL. Killing the group (not just the direct child) reaps helper children
+// that pandoc/pdftotext may fork, so a timeout cannot orphan grandchildren.
+func (p *LocalExtractParser) kill(cmd *exec.Cmd, done <-chan error) {
+	_ = killProcessGroup(cmd, syscallSIGTERM)
+	select {
+	case <-done:
+	case <-time.After(killGrace):
+		_ = killProcessGroup(cmd, syscallSIGKILL)
+		<-done
+	}
+}
+
+// cappedWriter buffers up to cap bytes and silently discards the rest, while
+// always reporting a full write. Discarding past the cap keeps the child from
+// blocking on a full pipe; overflow records that truncation occurred.
+type cappedWriter struct {
+	buf      bytes.Buffer
+	cap      int
+	overflow bool
+}
+
+func (w *cappedWriter) Write(p []byte) (int, error) {
+	if remaining := w.cap - w.buf.Len(); remaining > 0 {
+		if len(p) <= remaining {
+			w.buf.Write(p)
+		} else {
+			w.buf.Write(p[:remaining])
+			w.overflow = true
+		}
+	} else if len(p) > 0 {
+		w.overflow = true
+	}
+	return len(p), nil
+}