From 95d0799494c579dc3df3973bf1696c589f997528 Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:08:50 +0100 Subject: [PATCH] refactor(settings): tighten types on closed choices, containers and bounds Enum-like settings whose allowed values were only listed in a comment are now Literal types, so a typo fails at startup with a message naming the allowed values instead of falling through to a default with a warning (or, for VECTOR_STORE, failing on first use): AUTH_TYPE, VECTOR_STORE, STORAGE_TYPE, URL_STRATEGY, OCR_BACKEND, OCR_ENGINE, SANDBOX_BACKEND, DOC_PARSER_ENGINE, TTS_PROVIDER, STT_PROVIDER Each keeps a before-validator that strips and lower-cases the value, since the registries that consume them already lower-cased at the use site, and AUTH_TYPE maps the "None"/"none"/"" spellings a .env file carries to None (it was the string "None" before, which only worked because nothing compared against it). An empty TTS/STT provider still means "off". LLM_PROVIDER stays a plain str because providers are plugin-extensible. Containers are typed (dict[str, int], list[str], dict[str, Any]) instead of bare dict/list, six fields that were Optional with a non-None default are plain, and integer settings whose description already states a range carry it as a constraint (ge=0 for "0 disables", ge=1 for counts that cannot be zero, 0 < threshold <= 1). --- docs/content/Deploying/Settings-Reference.mdx | 72 +++++++++---------- docsgpt/core/settings/_shared.py | 9 ++- docsgpt/core/settings/agents.py | 9 +-- docsgpt/core/settings/auth.py | 16 +++-- docsgpt/core/settings/connectors.py | 4 +- docsgpt/core/settings/embeddings.py | 3 +- docsgpt/core/settings/events.py | 4 +- docsgpt/core/settings/guardrails.py | 6 +- docsgpt/core/settings/ingestion.py | 20 ++++-- docsgpt/core/settings/llm.py | 2 +- docsgpt/core/settings/ocr.py | 16 +++-- docsgpt/core/settings/retrieval.py | 19 +++-- docsgpt/core/settings/sandbox.py | 17 +++-- docsgpt/core/settings/server.py | 2 +- docsgpt/core/settings/speech.py | 19 +++-- docsgpt/core/settings/storage.py | 15 ++-- docsgpt/core/settings/vectorstores.py | 14 ++-- docsgpt/core/settings/workers.py | 3 +- tests/core/test_settings.py | 53 ++++++++++++++ 19 files changed, 206 insertions(+), 97 deletions(-) diff --git a/docs/content/Deploying/Settings-Reference.mdx b/docs/content/Deploying/Settings-Reference.mdx index 6b935de9..59b20430 100644 --- a/docs/content/Deploying/Settings-Reference.mdx +++ b/docs/content/Deploying/Settings-Reference.mdx @@ -20,9 +20,9 @@ How users authenticate: none, a shared token, per-session JWTs, or OIDC SSO. ### `AUTH_TYPE` -Type `str`, default unset. +Type `"simple_jwt" | "session_jwt" | "oidc"`, default unset. -Authentication mode: simple_jwt, session_jwt, oidc, or unset for no authentication. +Authentication mode: simple_jwt, session_jwt, oidc, or unset (None) for no authentication. ### `JWT_SECRET_KEY` @@ -86,7 +86,7 @@ Override for the callback URL; default is <request host>/api/auth/oidc/callba ### `OIDC_SESSION_LIFETIME_SECONDS` -Type `int`, default `28800`. +Type `int`, default `28800`, must be > 0. Lifetime of the minted session JWT in seconds (8h). @@ -354,13 +354,13 @@ Truncate each remote embed input to N tokens (overflow is lost). ### `EMBEDDINGS_BATCH_SIZE` -Type `int`, default `32`. +Type `int`, default `32`, must be >= 1. Chunks per store transaction and per remote embed request. ### `EMBEDDINGS_MODEL_BATCH_SIZE` -Type `int`, default `1`. +Type `int`, default `1`, must be >= 1. Documents per local ONNX forward pass. Each pass pads to its longest input, and that waste grows with the square of chunk length: at 1250 tokens, 32 peaked at 6.6 GB, 1 at 2.9 GB. @@ -413,9 +413,9 @@ Which vector store answers searches and how retrieval fans out across sources. ### `VECTOR_STORE` -Type `str`, default `faiss`. +Type `"faiss" | "elasticsearch" | "mongodb" | "qdrant" | "milvus" | "pgvector"`, default `faiss`. -Vector store backend: faiss, elasticsearch, mongodb, qdrant, milvus or pgvector. +Vector store backend. ### `RETRIEVERS_ENABLED` @@ -425,7 +425,7 @@ Retriever keys an agent may use; must match RetrieverCreator.retrievers registry ### `RETRIEVAL_MAX_PARALLEL_SOURCES` -Type `int`, default `4`. +Type `int`, default `4`, must be >= 1. Concurrent per-source searches in one retrieval; the query is embedded once and shared. @@ -580,7 +580,7 @@ pgvector connection string. postgres://, postgresql:// and postgresql+psycopg:// ### `PGVECTOR_POOL_MAX_SIZE` -Type `int`, default `8`. +Type `int`, default `8`, must be >= 0. Per-process connection pool size; 0 uses one direct connection per store. @@ -680,13 +680,13 @@ Broker visibility timeout in seconds. Must exceed the longest legitimate task ru ### `CELERY_WORKER_MAX_MEMORY_PER_CHILD` -Type `int`, default `4194304`. +Type `int`, default `4194304`, must be >= 0. Recycle a prefork child past this resident size in KB; backstops docling/torch heap growth. Checked between tasks, so it does not bound the peak within one. 0 disables. ### `CELERY_WORKER_MAX_TASKS_PER_CHILD` -Type `int`, default `0`. +Type `int`, default `0`, must be >= 0. Recycle a worker child after N tasks; 0 disables. @@ -763,7 +763,7 @@ Send images to a remote parser. ### `DOC_PARSER_ENGINE` -Type `str`, default `anydoc`. +Type `"anydoc" | "docling"`, default `anydoc`. Document parser for source ingestion, chat attachments and the read_document tool. "anydoc" (default): firecrawl-anydoc, a Rust converter with no ML models; milliseconds per file, ~100 MB peak RSS. "docling": the layout/table-model pipeline (optional install; needed for read_document's structured output and the docling OCR backend). Files anydoc cannot convert (scanned PDFs, malformed input) fall back to docling when it is installed, otherwise to the native OCR parsers (OCR on) or the legacy parsers. Rollback to the previous behaviour is this one variable. @@ -793,7 +793,7 @@ Largest HTML/XML docling will parse, in bytes. ### `MARKUP_MAX_BYTES` -Type `int`, default `8000000`. +Type `int`, default `8000000`, must be >= 0. HTML/XHTML larger than this (bytes) are head-truncated before the markdownify parser runs (the anydoc engine's HTML path). The tree that path builds costs ~50x the input (30 MB of HTML measured at 1.6 GB RSS) and the upload cap is 100 MB, so the gate is what keeps one upload from taking the ingest worker down. 0 disables it. @@ -841,13 +841,13 @@ Cap on the pixel count of an image passed to an agent. ### `GITHUB_INGEST_MAX_FILE_BYTES` -Type `int`, default `1048576`. +Type `int`, default `1048576`, must be >= 0. Skip GitHub repo blobs larger than this (0 = no cap). ### `GITHUB_INGEST_MAX_WORKERS` -Type `int`, default `8`. +Type `int`, default `8`, must be >= 1. Parallel file fetches per GitHub repo ingest. @@ -877,7 +877,7 @@ Absolute ceiling on the size-scaled parse window, in seconds. ### `DOCUMENT_PARSE_MAX_BYTES` -Type `int`, default `0`. +Type `int`, default `0`, must be >= 0. Cap on a parsed document's bytes (0 = reuse SANDBOX_MAX_INPUT_BYTES). @@ -912,13 +912,13 @@ OCR scanned PDFs and images attached to a chat. ### `OCR_BACKEND` -Type `str`, default `auto`. +Type `"auto" | "docling" | "native"`, default `auto`. Which stack runs OCR when it is on. auto: docling when installed, otherwise native. docling: the layout-model pipeline (hybrid region OCR, reading order, table structure); needs the optional docling extra. native: pypdfium2/Pillow page rendering straight into tesseract or a DeepSeek-OCR endpoint (docsgpt/parser/file/ocr_parser.py); no ML models in the worker, tables come out as text lines under tesseract. ### `OCR_ENGINE` -Type `str`, default `tesseract`. +Type `"tesseract" | "deepseek" | "auto" | "ocrmac" | "rapidocr"`, default `tesseract`. OCR engine used when OCR is on. Benched 2026-08 on EN/ZH/table/degraded scans (docs/Guides/ocr has the menu). tesseract (recommended): best classic-engine accuracy (perfect EN word recall, 0.000 bilingual CER, 100% table cells), ~35 MB, CPU-only; needs the system binary and language packs, an optional install like every OCR dependency (build with INSTALL_TESSERACT=true, or apt/brew install tesseract-ocr for a local run); both backends. deepseek: DeepSeek-OCR against an Ollama/vLLM endpoint (OCR_DEEPSEEK_*); best table/CJK quality, the worker stays light (no layout models) but each page costs seconds on the model server; both backends. auto: docling's pick, ocrmac on macOS (excellent), rapidocr on Linux (silently shreds some long text lines; avoid as a server default). ocrmac | rapidocr: force one of those. auto/ocrmac/rapidocr exist only inside docling; the native backend runs tesseract for them. An engine that is not installed degrades (docling: to auto) with a warning instead of failing the parse. @@ -954,7 +954,7 @@ Native backend only: resolution at which pages without a text layer are rendered ### `OCR_MIN_CHARS_PER_PAGE` -Type `int`, default `20`, also read from `DOCLING_OCR_MIN_CHARS_PER_PAGE`. +Type `int`, default `20`, must be >= 0, also read from `DOCLING_OCR_MIN_CHARS_PER_PAGE`. Chars-per-page floor below which an OCR'd PDF/image parse is treated as an OCR dropout rather than as content (long-running docling workers were observed returning zero characters for every scanned page after a long scanned PDF, with no error). docling retries once on a fresh full-page-OCR converter; both backends then fail loudly instead of indexing an empty document. 0 disables the guard. @@ -965,13 +965,13 @@ Local disk or an S3-compatible bucket, and how download URLs are produced. ### `STORAGE_TYPE` -Type `str`, default `local`. +Type `"local" | "s3"`, default `local`. -File storage backend: local or s3. +File storage backend. ### `URL_STRATEGY` -Type `str`, default `backend`. +Type `"backend" | "s3"`, default `backend`. How download links are produced: backend (streamed through the API) or s3 (presigned URLs). @@ -1143,7 +1143,7 @@ Bounds uvicorn's shutdown drain (uvicorn_worker doesn't forward --graceful-timeo ### `WSGI_THREADPOOL_WORKERS` -Type `int`, default `96`. +Type `int`, default `96`, must be >= 1. Threads serving the WSGI (Flask) part of the app under the ASGI server. @@ -1166,7 +1166,7 @@ Internal SSE push channel (notifications and durable replay journal). False make ### `EVENTS_STREAM_MAXLEN` -Type `int`, default `1000`. +Type `int`, default `1000`, must be >= 1. Per-user durable backlog cap in entries; ~24h of replay at typical rates. @@ -1178,7 +1178,7 @@ Interval between SSE keepalive comments. ### `SSE_MAX_CONCURRENT_PER_USER` -Type `int`, default `8`. +Type `int`, default `8`, must be >= 0. Simultaneous SSE connections per user; each holds a pooled async Redis connection for its lifetime. 8 covers multi-tab use without one user starving the pool. 0 disables. @@ -1190,7 +1190,7 @@ Pool size of the async Redis client behind the event-loop routes, per process. E ### `EVENTS_REPLAY_MAX_PER_REQUEST` -Type `int`, default `200`. +Type `int`, default `200`, must be >= 1. Backlog entries XRANGE returns per /api/events snapshot. Bounds what one replay moves from Redis to the wire: a client looping Last-Event-ID reconnects enumerates at most this many per round-trip. @@ -1291,7 +1291,7 @@ Pre-fetch retrieval before the agent's first turn. ### `TOOL_RESULT_MAX_TOKENS` -Type `int`, default `20000`. +Type `int`, default `20000`, must be >= 0. Cap on one tool result entering the LLM context (0 disables); journal and DB keep it whole. @@ -1303,7 +1303,7 @@ Compress long conversations once they approach the context window. ### `COMPRESSION_THRESHOLD_PERCENTAGE` -Type `float`, default `0.8`. +Type `float`, default `0.8`, must be > 0 and <= 1. Fraction of the context window at which compression triggers. @@ -1327,7 +1327,7 @@ Keep only the last N compression points to prevent DB bloat. ### `COMPRESSION_RECENT_FIELD_MAX_TOKENS` -Type `int`, default `8000`. +Type `int`, default `8000`, must be >= 0. Per-field cap on the verbatim tail kept after a compression point (0 disables). @@ -1474,7 +1474,7 @@ The app is a CLIENT of an always-on runner; defaults are safe so app import neve ### `SANDBOX_BACKEND` -Type `str`, default `jupyter`. +Type `"jupyter" | "daytona"`, default `jupyter`. Sandbox backend: jupyter (self-host) or daytona (Daytona Cloud). @@ -1582,13 +1582,13 @@ Default runtime language for created sandboxes. ### `DAYTONA_AUTO_STOP_INTERVAL` -Type `int`, default `15`. +Type `int`, default `15`, must be >= 0. Minutes idle before Daytona auto-stops a sandbox (0 disables). ### `DAYTONA_AUTO_DELETE_INTERVAL` -Type `int`, default `60`. +Type `int`, default `60`, must be >= -1. Minutes after stop before Daytona auto-deletes a sandbox (-1 disables). @@ -1605,9 +1605,9 @@ Voice providers and transcription options. ### `TTS_PROVIDER` -Type `str`, default `google_tts`. +Type `"google_tts" | "elevenlabs" | "none"`, default `google_tts`. -Text-to-speech provider: google_tts, elevenlabs, or none to switch it off. +Text-to-speech provider; none switches it off. ### `ELEVENLABS_API_KEY` @@ -1617,9 +1617,9 @@ ElevenLabs API key. ### `STT_PROVIDER` -Type `str`, default `openai`. +Type `"openai" | "faster_whisper" | "none"`, default `openai`. -Speech-to-text provider: openai, faster_whisper, or none to switch it off. +Speech-to-text provider; none switches it off. ### `OPENAI_STT_MODEL` diff --git a/docsgpt/core/settings/_shared.py b/docsgpt/core/settings/_shared.py index 4eb5c48d..235ee07b 100644 --- a/docsgpt/core/settings/_shared.py +++ b/docsgpt/core/settings/_shared.py @@ -9,7 +9,7 @@ domain's definitions live in their own module. from __future__ import annotations -from typing import Optional +from typing import Any, Optional from pydantic_settings import BaseSettings, SettingsConfigDict @@ -20,6 +20,13 @@ class SettingsGroup(BaseSettings): model_config = SettingsConfigDict(extra="ignore") +def normalize_choice(value: Any) -> Any: + """Case-fold a closed-choice setting so ``PGVector`` and ``pgvector`` are the same choice.""" + if isinstance(value, str): + return value.strip().lower() + return value + + def normalize_secret(value: Optional[str]) -> Optional[str]: """Map the ways an unset secret reaches us from ``.env`` to ``None``. diff --git a/docsgpt/core/settings/agents.py b/docsgpt/core/settings/agents.py index fdee1cc3..9b682786 100644 --- a/docsgpt/core/settings/agents.py +++ b/docsgpt/core/settings/agents.py @@ -14,11 +14,11 @@ class AgentSettings(SettingsGroup): AGENT_NAME: str = Field(default="classic", description="Default agent type for agentless chats.") DEFAULT_MAX_HISTORY: int = Field(default=150, description="Default number of history messages kept.") - DEFAULT_AGENT_LIMITS: dict = Field( + DEFAULT_AGENT_LIMITS: dict[str, int] = Field( default={"token_limit": 50000, "request_limit": 500}, description="Per-agent default quotas: tokens and requests.", ) - DEFAULT_CHAT_TOOLS: list = Field( + DEFAULT_CHAT_TOOLS: list[str] = Field( default=["memory", "read_webpage", "scheduler"], description=( "Config-free tools on by default in agentless chats. scheduler is dual-registered in " @@ -30,6 +30,7 @@ class AgentSettings(SettingsGroup): ENABLE_TOOL_PREFETCH: bool = Field(default=True, description="Pre-fetch retrieval before the agent's first turn.") TOOL_RESULT_MAX_TOKENS: int = Field( default=20000, + ge=0, description="Cap on one tool result entering the LLM context (0 disables); journal and DB keep it whole.", ) @@ -38,7 +39,7 @@ class AgentSettings(SettingsGroup): default=True, description="Compress long conversations once they approach the context window." ) COMPRESSION_THRESHOLD_PERCENTAGE: float = Field( - default=0.8, description="Fraction of the context window at which compression triggers." + default=0.8, gt=0, le=1, description="Fraction of the context window at which compression triggers." ) COMPRESSION_MODEL_OVERRIDE: Optional[str] = Field( default=None, description="Use a different model for compression; unset reuses the answer model." @@ -48,7 +49,7 @@ class AgentSettings(SettingsGroup): default=3, description="Keep only the last N compression points to prevent DB bloat." ) COMPRESSION_RECENT_FIELD_MAX_TOKENS: int = Field( - default=8000, description="Per-field cap on the verbatim tail kept after a compression point (0 disables)." + default=8000, ge=0, description="Per-field cap on the verbatim tail kept after a compression point (0 disables)." ) # Workflows. diff --git a/docsgpt/core/settings/auth.py b/docsgpt/core/settings/auth.py index d0763934..10e69801 100644 --- a/docsgpt/core/settings/auth.py +++ b/docsgpt/core/settings/auth.py @@ -2,19 +2,19 @@ from __future__ import annotations -from typing import Optional +from typing import Literal, Optional from pydantic import Field, field_validator -from docsgpt.core.settings._shared import SettingsGroup, normalize_secret +from docsgpt.core.settings._shared import SettingsGroup, normalize_choice, normalize_secret class AuthSettings(SettingsGroup): """How users authenticate: none, a shared token, per-session JWTs, or OIDC SSO.""" - AUTH_TYPE: Optional[str] = Field( + AUTH_TYPE: Optional[Literal["simple_jwt", "session_jwt", "oidc"]] = Field( default=None, - description="Authentication mode: simple_jwt, session_jwt, oidc, or unset for no authentication.", + description="Authentication mode: simple_jwt, session_jwt, oidc, or unset (None) for no authentication.", ) JWT_SECRET_KEY: str = Field( default="", @@ -51,7 +51,7 @@ class AuthSettings(SettingsGroup): default=None, description="Override for the callback URL; default is /api/auth/oidc/callback." ) OIDC_SESSION_LIFETIME_SECONDS: int = Field( - default=28800, description="Lifetime of the minted session JWT in seconds (8h)." + default=28800, gt=0, description="Lifetime of the minted session JWT in seconds (8h)." ) OIDC_PROVIDER_NAME: Optional[str] = Field( default=None, description='Sign-in button label, e.g. "Acme SSO".' @@ -85,3 +85,9 @@ class AuthSettings(SettingsGroup): @classmethod def _normalize_auth_secrets(cls, v): return normalize_secret(v) + + @field_validator("AUTH_TYPE", mode="before") + @classmethod + def _normalize_auth_type(cls, v): + # ``AUTH_TYPE=None`` and ``AUTH_TYPE=`` in .env both mean "no authentication". + return normalize_choice(normalize_secret(v)) diff --git a/docsgpt/core/settings/connectors.py b/docsgpt/core/settings/connectors.py index 303c00b6..676ffe64 100644 --- a/docsgpt/core/settings/connectors.py +++ b/docsgpt/core/settings/connectors.py @@ -15,7 +15,7 @@ class ConnectorSettings(SettingsGroup): # Google Drive integration. GOOGLE_CLIENT_ID: Optional[str] = Field(default=None, description="Google OAuth client id.") GOOGLE_CLIENT_SECRET: Optional[str] = Field(default=None, description="Google OAuth client secret.") - CONNECTOR_REDIRECT_BASE_URI: Optional[str] = Field( + CONNECTOR_REDIRECT_BASE_URI: str = Field( default="http://127.0.0.1:7091/api/connectors/callback", description="OAuth callback URL; register it as-is in your provider's console (e.g. GCP).", ) @@ -31,7 +31,7 @@ class ConnectorSettings(SettingsGroup): # Microsoft Entra ID (Azure AD) integration. MICROSOFT_CLIENT_ID: Optional[str] = Field(default=None, description="Azure AD application (client) id.") MICROSOFT_CLIENT_SECRET: Optional[str] = Field(default=None, description="Azure AD application client secret.") - MICROSOFT_TENANT_ID: Optional[str] = Field( + MICROSOFT_TENANT_ID: str = Field( default="common", description="Azure AD tenant id, or 'common' for multi-tenant." ) MICROSOFT_AUTHORITY: Optional[str] = Field( diff --git a/docsgpt/core/settings/embeddings.py b/docsgpt/core/settings/embeddings.py index 85de4d15..3ac749aa 100644 --- a/docsgpt/core/settings/embeddings.py +++ b/docsgpt/core/settings/embeddings.py @@ -32,10 +32,11 @@ class EmbeddingsSettings(SettingsGroup): default=None, description="Truncate each remote embed input to N tokens (overflow is lost)." ) EMBEDDINGS_BATCH_SIZE: int = Field( - default=32, description="Chunks per store transaction and per remote embed request." + default=32, ge=1, description="Chunks per store transaction and per remote embed request." ) EMBEDDINGS_MODEL_BATCH_SIZE: int = Field( default=1, + ge=1, description=( "Documents per local ONNX forward pass. Each pass pads to its longest input, and that waste grows " "with the square of chunk length: at 1250 tokens, 32 peaked at 6.6 GB, 1 at 2.9 GB." diff --git a/docsgpt/core/settings/events.py b/docsgpt/core/settings/events.py index 0ab01d4c..f8ca6742 100644 --- a/docsgpt/core/settings/events.py +++ b/docsgpt/core/settings/events.py @@ -18,11 +18,12 @@ class EventsSettings(SettingsGroup): ), ) EVENTS_STREAM_MAXLEN: int = Field( - default=1000, description="Per-user durable backlog cap in entries; ~24h of replay at typical rates." + default=1000, ge=1, description="Per-user durable backlog cap in entries; ~24h of replay at typical rates." ) SSE_KEEPALIVE_SECONDS: int = Field(default=15, ge=1, description="Interval between SSE keepalive comments.") SSE_MAX_CONCURRENT_PER_USER: int = Field( default=8, + ge=0, description=( "Simultaneous SSE connections per user; each holds a pooled async Redis connection for its lifetime. " "8 covers multi-tab use without one user starving the pool. 0 disables." @@ -40,6 +41,7 @@ class EventsSettings(SettingsGroup): ) EVENTS_REPLAY_MAX_PER_REQUEST: int = Field( default=200, + ge=1, description=( "Backlog entries XRANGE returns per /api/events snapshot. Bounds what one replay moves from Redis to " "the wire: a client looping Last-Event-ID reconnects enumerates at most this many per round-trip." diff --git a/docsgpt/core/settings/guardrails.py b/docsgpt/core/settings/guardrails.py index 81ac60f2..871f748b 100644 --- a/docsgpt/core/settings/guardrails.py +++ b/docsgpt/core/settings/guardrails.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Optional +from typing import Any, Optional from pydantic import Field @@ -13,10 +13,10 @@ class GuardrailSettings(SettingsGroup): """Input/output checks every agent runs, and the floor no agent may weaken.""" GUARDRAILS_ENABLED: bool = Field(default=True, description="Master switch; False disables every stage.") - GUARDRAILS_CHECKS_ENABLED: list = Field( + GUARDRAILS_CHECKS_ENABLED: list[str] = Field( default=[], description="Allowlist of GuardrailCreator.checks keys; empty means every registered check." ) - GUARDRAILS_FLOOR: dict = Field( + GUARDRAILS_FLOOR: dict[str, Any] = Field( default={}, description=( "A GuardrailsConfig fragment every agent inherits and cannot weaken; agents may add controls or " diff --git a/docsgpt/core/settings/ingestion.py b/docsgpt/core/settings/ingestion.py index 38da376d..d44a1360 100644 --- a/docsgpt/core/settings/ingestion.py +++ b/docsgpt/core/settings/ingestion.py @@ -2,9 +2,11 @@ from __future__ import annotations -from pydantic import Field +from typing import Literal -from docsgpt.core.settings._shared import SettingsGroup +from pydantic import Field, field_validator + +from docsgpt.core.settings._shared import SettingsGroup, normalize_choice class IngestionSettings(SettingsGroup): @@ -37,7 +39,7 @@ class IngestionSettings(SettingsGroup): ) PARSE_PDF_AS_IMAGE: bool = Field(default=False, description="Render PDF pages to images before parsing.") PARSE_IMAGE_REMOTE: bool = Field(default=False, description="Send images to a remote parser.") - DOC_PARSER_ENGINE: str = Field( + DOC_PARSER_ENGINE: Literal["anydoc", "docling"] = Field( default="anydoc", description=( 'Document parser for source ingestion, chat attachments and the read_document tool. "anydoc" ' @@ -66,6 +68,7 @@ class IngestionSettings(SettingsGroup): ) MARKUP_MAX_BYTES: int = Field( default=8_000_000, + ge=0, description=( "HTML/XHTML larger than this (bytes) are head-truncated before the markdownify parser runs (the " "anydoc engine's HTML path). The tree that path builds costs ~50x the input (30 MB of HTML measured " @@ -113,9 +116,9 @@ class IngestionSettings(SettingsGroup): default=16_777_216, description="Cap on the pixel count of an image passed to an agent." ) GITHUB_INGEST_MAX_FILE_BYTES: int = Field( - default=1048576, description="Skip GitHub repo blobs larger than this (0 = no cap)." + default=1048576, ge=0, description="Skip GitHub repo blobs larger than this (0 = no cap)." ) - GITHUB_INGEST_MAX_WORKERS: int = Field(default=8, description="Parallel file fetches per GitHub repo ingest.") + GITHUB_INGEST_MAX_WORKERS: int = Field(default=8, ge=1, description="Parallel file fetches per GitHub repo ingest.") # read_document parsing on a dedicated Celery queue (backend parser). DOCUMENT_PARSE_QUEUE: str = Field(default="parsing", description="Celery queue the parse_document task is routed to.") @@ -134,7 +137,7 @@ class IngestionSettings(SettingsGroup): default=900, description="Absolute ceiling on the size-scaled parse window, in seconds." ) DOCUMENT_PARSE_MAX_BYTES: int = Field( - default=0, description="Cap on a parsed document's bytes (0 = reuse SANDBOX_MAX_INPUT_BYTES)." + default=0, ge=0, description="Cap on a parsed document's bytes (0 = reuse SANDBOX_MAX_INPUT_BYTES)." ) DOCUMENT_MAX_DECOMPRESSED_BYTES: int = Field( default=300 * 1024 * 1024, description="Cap on bytes decompressed from an archive handed to read_document." @@ -142,3 +145,8 @@ class IngestionSettings(SettingsGroup): DOCUMENT_MAX_ARCHIVE_ENTRIES: int = Field( default=10000, description="Cap on entries in an archive handed to read_document." ) + + @field_validator("DOC_PARSER_ENGINE", mode="before") + @classmethod + def _normalize_parser_engine(cls, v): + return normalize_choice(v) diff --git a/docsgpt/core/settings/llm.py b/docsgpt/core/settings/llm.py index 232ad3be..11c17cac 100644 --- a/docsgpt/core/settings/llm.py +++ b/docsgpt/core/settings/llm.py @@ -59,7 +59,7 @@ class LLMSettings(SettingsGroup): DEFAULT_LLM_TOKEN_LIMIT: int = Field( default=128000, description="Context window assumed when the model is not found in the registry." ) - RESERVED_TOKENS: dict = Field( + RESERVED_TOKENS: dict[str, int] = Field( default={"system_prompt": 500, "current_query": 500, "safety_buffer": 1000}, description="Tokens held back from the context window for the system prompt, the query and a safety buffer.", ) diff --git a/docsgpt/core/settings/ocr.py b/docsgpt/core/settings/ocr.py index 264b21c5..2ffc0432 100644 --- a/docsgpt/core/settings/ocr.py +++ b/docsgpt/core/settings/ocr.py @@ -2,9 +2,11 @@ from __future__ import annotations -from pydantic import AliasChoices, Field +from typing import Literal -from docsgpt.core.settings._shared import SettingsGroup +from pydantic import AliasChoices, Field, field_validator + +from docsgpt.core.settings._shared import SettingsGroup, normalize_choice class OCRSettings(SettingsGroup): @@ -25,7 +27,7 @@ class OCRSettings(SettingsGroup): validation_alias=AliasChoices("OCR_ATTACHMENTS_ENABLED", "DOCLING_OCR_ATTACHMENTS_ENABLED"), description="OCR scanned PDFs and images attached to a chat.", ) - OCR_BACKEND: str = Field( + OCR_BACKEND: Literal["auto", "docling", "native"] = Field( default="auto", description=( "Which stack runs OCR when it is on. auto: docling when installed, otherwise native. docling: the " @@ -35,7 +37,7 @@ class OCRSettings(SettingsGroup): "lines under tesseract." ), ) - OCR_ENGINE: str = Field( + OCR_ENGINE: Literal["tesseract", "deepseek", "auto", "ocrmac", "rapidocr"] = Field( default="tesseract", description=( "OCR engine used when OCR is on. Benched 2026-08 on EN/ZH/table/degraded scans (docs/Guides/ocr has " @@ -80,6 +82,7 @@ class OCRSettings(SettingsGroup): ) OCR_MIN_CHARS_PER_PAGE: int = Field( default=20, + ge=0, validation_alias=AliasChoices("OCR_MIN_CHARS_PER_PAGE", "DOCLING_OCR_MIN_CHARS_PER_PAGE"), description=( "Chars-per-page floor below which an OCR'd PDF/image parse is treated as an OCR dropout rather than " @@ -89,3 +92,8 @@ class OCRSettings(SettingsGroup): "guard." ), ) + + @field_validator("OCR_BACKEND", "OCR_ENGINE", mode="before") + @classmethod + def _normalize_ocr_choices(cls, v): + return normalize_choice(v) diff --git a/docsgpt/core/settings/retrieval.py b/docsgpt/core/settings/retrieval.py index 39672d29..a0bf87f9 100644 --- a/docsgpt/core/settings/retrieval.py +++ b/docsgpt/core/settings/retrieval.py @@ -2,21 +2,20 @@ from __future__ import annotations -from typing import Optional +from typing import Literal, Optional -from pydantic import Field +from pydantic import Field, field_validator -from docsgpt.core.settings._shared import SettingsGroup +from docsgpt.core.settings._shared import SettingsGroup, normalize_choice class RetrievalSettings(SettingsGroup): """Which vector store answers searches and how retrieval fans out across sources.""" - VECTOR_STORE: str = Field( - default="faiss", - description="Vector store backend: faiss, elasticsearch, mongodb, qdrant, milvus or pgvector.", + VECTOR_STORE: Literal["faiss", "elasticsearch", "mongodb", "qdrant", "milvus", "pgvector"] = Field( + default="faiss", description="Vector store backend." ) - RETRIEVERS_ENABLED: list = Field( + RETRIEVERS_ENABLED: list[str] = Field( default=["classic", "default"], description=( "Retriever keys an agent may use; must match RetrieverCreator.retrievers registry keys, NOT the " @@ -25,6 +24,7 @@ class RetrievalSettings(SettingsGroup): ) RETRIEVAL_MAX_PARALLEL_SOURCES: int = Field( default=4, + ge=1, description="Concurrent per-source searches in one retrieval; the query is embedded once and shared.", ) PER_SOURCE_RETRIEVAL_ENABLED: bool = Field( @@ -38,3 +38,8 @@ class RetrievalSettings(SettingsGroup): GRAPHRAG_MAX_CHUNKS_FOR_EXTRACTION: int = Field( default=2000, description="Hard cap on chunks extracted per source (cost control)." ) + + @field_validator("VECTOR_STORE", mode="before") + @classmethod + def _normalize_vector_store(cls, v): + return normalize_choice(v) diff --git a/docsgpt/core/settings/sandbox.py b/docsgpt/core/settings/sandbox.py index 958734bc..96e4d6e0 100644 --- a/docsgpt/core/settings/sandbox.py +++ b/docsgpt/core/settings/sandbox.py @@ -2,17 +2,17 @@ from __future__ import annotations -from typing import Optional +from typing import Literal, Optional -from pydantic import Field +from pydantic import Field, field_validator -from docsgpt.core.settings._shared import SettingsGroup +from docsgpt.core.settings._shared import SettingsGroup, normalize_choice class SandboxSettings(SettingsGroup): """The app is a CLIENT of an always-on runner; defaults are safe so app import never fails unconfigured.""" - SANDBOX_BACKEND: str = Field( + SANDBOX_BACKEND: Literal["jupyter", "daytona"] = Field( default="jupyter", description="Sandbox backend: jupyter (self-host) or daytona (Daytona Cloud)." ) SANDBOX_GATEWAY_URL: str = Field( @@ -78,11 +78,16 @@ class SandboxSettings(SettingsGroup): ) DAYTONA_LANGUAGE: str = Field(default="python", description="Default runtime language for created sandboxes.") DAYTONA_AUTO_STOP_INTERVAL: int = Field( - default=15, description="Minutes idle before Daytona auto-stops a sandbox (0 disables)." + default=15, ge=0, description="Minutes idle before Daytona auto-stops a sandbox (0 disables)." ) DAYTONA_AUTO_DELETE_INTERVAL: int = Field( - default=60, description="Minutes after stop before Daytona auto-deletes a sandbox (-1 disables)." + default=60, ge=-1, description="Minutes after stop before Daytona auto-deletes a sandbox (-1 disables)." ) DAYTONA_MAX_SANDBOXES: int = Field( default=50, description="Cap on concurrent live Daytona sandboxes (cost-DoS guard)." ) + + @field_validator("SANDBOX_BACKEND", mode="before") + @classmethod + def _normalize_sandbox_backend(cls, v): + return normalize_choice(v) diff --git a/docsgpt/core/settings/server.py b/docsgpt/core/settings/server.py index 84bb2510..afb79121 100644 --- a/docsgpt/core/settings/server.py +++ b/docsgpt/core/settings/server.py @@ -28,7 +28,7 @@ class ServerSettings(SettingsGroup): ), ) WSGI_THREADPOOL_WORKERS: int = Field( - default=96, description="Threads serving the WSGI (Flask) part of the app under the ASGI server." + default=96, ge=1, description="Threads serving the WSGI (Flask) part of the app under the ASGI server." ) V1_SESSION_TTL_SECONDS: int = Field( default=24 * 60 * 60, diff --git a/docsgpt/core/settings/speech.py b/docsgpt/core/settings/speech.py index 465c8fc5..da5c245e 100644 --- a/docsgpt/core/settings/speech.py +++ b/docsgpt/core/settings/speech.py @@ -2,22 +2,22 @@ from __future__ import annotations -from typing import Optional +from typing import Literal, Optional from pydantic import Field, field_validator -from docsgpt.core.settings._shared import SettingsGroup, normalize_secret +from docsgpt.core.settings._shared import SettingsGroup, normalize_choice, normalize_secret class SpeechSettings(SettingsGroup): """Voice providers and transcription options.""" - TTS_PROVIDER: str = Field( - default="google_tts", description="Text-to-speech provider: google_tts, elevenlabs, or none to switch it off." + TTS_PROVIDER: Literal["google_tts", "elevenlabs", "none"] = Field( + default="google_tts", description="Text-to-speech provider; none switches it off." ) ELEVENLABS_API_KEY: Optional[str] = Field(default=None, description="ElevenLabs API key.") - STT_PROVIDER: str = Field( - default="openai", description="Speech-to-text provider: openai, faster_whisper, or none to switch it off." + STT_PROVIDER: Literal["openai", "faster_whisper", "none"] = Field( + default="openai", description="Speech-to-text provider; none switches it off." ) OPENAI_STT_MODEL: str = Field(default="gpt-4o-mini-transcribe", description="OpenAI transcription model.") STT_LANGUAGE: Optional[str] = Field(default=None, description="Language hint for transcription; unset auto-detects.") @@ -29,3 +29,10 @@ class SpeechSettings(SettingsGroup): @classmethod def _normalize_speech_secrets(cls, v): return normalize_secret(v) + + @field_validator("TTS_PROVIDER", "STT_PROVIDER", mode="before") + @classmethod + def _normalize_speech_providers(cls, v): + # An empty value has always meant "off"; keep that spelling working. + v = normalize_choice(v) + return "none" if v == "" else v diff --git a/docsgpt/core/settings/storage.py b/docsgpt/core/settings/storage.py index ac970a54..2fe75347 100644 --- a/docsgpt/core/settings/storage.py +++ b/docsgpt/core/settings/storage.py @@ -2,18 +2,18 @@ from __future__ import annotations -from typing import Optional +from typing import Literal, Optional -from pydantic import Field +from pydantic import Field, field_validator -from docsgpt.core.settings._shared import SettingsGroup +from docsgpt.core.settings._shared import SettingsGroup, normalize_choice class StorageSettings(SettingsGroup): """Local disk or an S3-compatible bucket, and how download URLs are produced.""" - STORAGE_TYPE: str = Field(default="local", description="File storage backend: local or s3.") - URL_STRATEGY: str = Field( + STORAGE_TYPE: Literal["local", "s3"] = Field(default="local", description="File storage backend.") + URL_STRATEGY: Literal["backend", "s3"] = Field( default="backend", description="How download links are produced: backend (streamed through the API) or s3 (presigned URLs).", ) @@ -47,3 +47,8 @@ class StorageSettings(SettingsGroup): "S3_SECRET_ACCESS_KEY." ), ) + + @field_validator("STORAGE_TYPE", "URL_STRATEGY", mode="before") + @classmethod + def _normalize_storage_choices(cls, v): + return normalize_choice(v) diff --git a/docsgpt/core/settings/vectorstores.py b/docsgpt/core/settings/vectorstores.py index 13c504df..067d1df0 100644 --- a/docsgpt/core/settings/vectorstores.py +++ b/docsgpt/core/settings/vectorstores.py @@ -27,13 +27,13 @@ class VectorStoreSettings(SettingsGroup): ELASTIC_USERNAME: Optional[str] = Field(default=None, description="Elasticsearch username.") ELASTIC_PASSWORD: Optional[str] = Field(default=None, description="Elasticsearch password.") ELASTIC_URL: Optional[str] = Field(default=None, description="Elasticsearch URL.") - ELASTIC_INDEX: Optional[str] = Field(default="docsgpt", description="Elasticsearch index name.") + ELASTIC_INDEX: str = Field(default="docsgpt", description="Elasticsearch index name.") # Qdrant. - QDRANT_COLLECTION_NAME: Optional[str] = Field(default="docsgpt", description="Qdrant collection name.") + QDRANT_COLLECTION_NAME: str = Field(default="docsgpt", description="Qdrant collection name.") QDRANT_LOCATION: Optional[str] = Field(default=None, description="Qdrant location (':memory:' or a URL).") QDRANT_URL: Optional[str] = Field(default=None, description="Qdrant server URL.") - QDRANT_PORT: Optional[int] = Field(default=6333, description="Qdrant REST port.") + QDRANT_PORT: int = Field(default=6333, description="Qdrant REST port.") QDRANT_GRPC_PORT: int = Field(default=6334, description="Qdrant gRPC port.") QDRANT_PREFER_GRPC: bool = Field(default=False, description="Use gRPC instead of REST where possible.") QDRANT_HTTPS: Optional[bool] = Field(default=None, description="Use HTTPS for the Qdrant connection.") @@ -53,7 +53,7 @@ class VectorStoreSettings(SettingsGroup): ), ) PGVECTOR_POOL_MAX_SIZE: int = Field( - default=8, description="Per-process connection pool size; 0 uses one direct connection per store." + default=8, ge=0, description="Per-process connection pool size; 0 uses one direct connection per store." ) PGVECTOR_IVFFLAT_PROBES: Optional[int] = Field( default=None, @@ -61,7 +61,7 @@ class VectorStoreSettings(SettingsGroup): ) # Milvus. - MILVUS_COLLECTION_NAME: Optional[str] = Field(default="docsgpt", description="Milvus collection name.") + MILVUS_COLLECTION_NAME: str = Field(default="docsgpt", description="Milvus collection name.") MILVUS_URI: Optional[str] = Field( default_factory=lambda: str(home_dir() / "milvus_local.db"), description=( @@ -69,14 +69,14 @@ class VectorStoreSettings(SettingsGroup): "like the other local stores." ), ) - MILVUS_TOKEN: Optional[str] = Field(default="", description="Milvus auth token.") + MILVUS_TOKEN: str = Field(default="", description="Milvus auth token.") # LanceDB. LANCEDB_PATH: str = Field( default_factory=lambda: str(home_dir() / "data" / "lancedb"), description="LanceDB local data directory.", ) - LANCEDB_TABLE_NAME: Optional[str] = Field(default="docsgpts", description="LanceDB table for stored vectors.") + LANCEDB_TABLE_NAME: str = Field(default="docsgpts", description="LanceDB table for stored vectors.") @field_validator("PGVECTOR_CONNECTION_STRING", mode="before") @classmethod diff --git a/docsgpt/core/settings/workers.py b/docsgpt/core/settings/workers.py index fbc7daae..86246cab 100644 --- a/docsgpt/core/settings/workers.py +++ b/docsgpt/core/settings/workers.py @@ -24,13 +24,14 @@ class WorkerSettings(SettingsGroup): ) CELERY_WORKER_MAX_MEMORY_PER_CHILD: int = Field( default=4194304, + ge=0, description=( "Recycle a prefork child past this resident size in KB; backstops docling/torch heap growth. " "Checked between tasks, so it does not bound the peak within one. 0 disables." ), ) CELERY_WORKER_MAX_TASKS_PER_CHILD: int = Field( - default=0, description="Recycle a worker child after N tasks; 0 disables." + default=0, ge=0, description="Recycle a worker child after N tasks; 0 disables." ) API_URL: str = Field( default="http://localhost:7091", description="Backend URL the Celery worker calls back into." diff --git a/tests/core/test_settings.py b/tests/core/test_settings.py index 9c2f7d88..a4d64bf3 100644 --- a/tests/core/test_settings.py +++ b/tests/core/test_settings.py @@ -9,6 +9,7 @@ tracks the definitions. from pathlib import Path import pytest +from pydantic import ValidationError from docsgpt.core.settings import SETTINGS_GROUPS, Settings, settings from docsgpt.core.settings.reference import reference_path, render_reference @@ -107,3 +108,55 @@ class TestReference: "docs/content/Deploying/Settings-Reference.mdx is stale; " "run: python -m docsgpt.core.settings.reference --write" ) + + +@pytest.mark.unit +class TestClosedChoices: + """Enum-like settings are Literal types: a typo fails at startup instead of falling through.""" + + @pytest.mark.parametrize("raw", ["None", "none", "", " "]) + def test_auth_type_unset_spellings(self, raw): + assert Settings.model_validate({"AUTH_TYPE": raw}).AUTH_TYPE is None + + @pytest.mark.parametrize( + ("name", "raw", "expected"), + [ + ("AUTH_TYPE", " OIDC ", "oidc"), + ("VECTOR_STORE", "PGVector", "pgvector"), + ("STORAGE_TYPE", "S3", "s3"), + ("URL_STRATEGY", "Backend", "backend"), + ("OCR_BACKEND", "Native", "native"), + ("OCR_ENGINE", "Tesseract ", "tesseract"), + ("SANDBOX_BACKEND", "Daytona", "daytona"), + ("DOC_PARSER_ENGINE", "Docling", "docling"), + ("TTS_PROVIDER", "ElevenLabs", "elevenlabs"), + ("STT_PROVIDER", "", "none"), + ("TTS_PROVIDER", "NONE", "none"), + ], + ) + def test_choices_are_case_insensitive(self, name, raw, expected): + assert getattr(Settings.model_validate({name: raw}), name) == expected + + @pytest.mark.parametrize( + ("name", "raw"), + [ + ("AUTH_TYPE", "basic"), + ("VECTOR_STORE", "lancedb"), + ("STORAGE_TYPE", "gcs"), + ("OCR_BACKEND", "paddle"), + ("SANDBOX_BACKEND", "docker"), + ("DOC_PARSER_ENGINE", "fast"), + ("STT_PROVIDER", "whisper"), + ], + ) + def test_unknown_choice_is_rejected(self, name, raw): + with pytest.raises(ValidationError): + Settings.model_validate({name: raw}) + + @pytest.mark.parametrize( + ("name", "raw"), + [("EMBEDDINGS_BATCH_SIZE", 0), ("COMPRESSION_THRESHOLD_PERCENTAGE", 1.5), ("UPLOAD_MAX_FILE_BYTES", 0)], + ) + def test_out_of_range_numbers_are_rejected(self, name, raw): + with pytest.raises(ValidationError): + Settings.model_validate({name: raw})