diff --git a/application/parser/chunking_strategies.py b/application/parser/chunking_strategies.py
index e59cdd1b..87b40839 100644
--- a/application/parser/chunking_strategies.py
+++ b/application/parser/chunking_strategies.py
@@ -4,11 +4,12 @@ Each strategy honours ``max_tokens`` / ``min_tokens`` and reuses the classic
``Chunker``'s tiktoken encoding for token counting, so token budgets stay
consistent across strategies. Selecting a strategy is ingest-time only;
changing it requires a re-ingest (D8). Registered keys: ``recursive``,
-``markdown``, ``parent_child``. ``semantic`` is intentionally deferred.
+``markdown``, ``parent_child``, ``semantic``.
"""
from __future__ import annotations
+import logging
import re
from typing import List
@@ -17,6 +18,8 @@ from application.parser.chunking_creator import ChunkerCreator
from application.parser.schema.base import Document
from application.utils import get_encoding
+logger = logging.getLogger(__name__)
+
class _BaseStrategyChunker:
"""Shared token helpers for strategy chunkers.
@@ -64,6 +67,25 @@ class _BaseStrategyChunker:
},
)
+ def _merge_to_min(self, pieces: List[str], joiner: str) -> List[str]:
+ """Accumulate pieces up to ``max_tokens``, flushing past ``min_tokens``."""
+ merged: List[str] = []
+ buffer = ""
+ for piece in pieces:
+ candidate = f"{buffer}{joiner}{piece}" if buffer else piece
+ if self._token_count(candidate) <= self.max_tokens:
+ buffer = candidate
+ else:
+ if buffer:
+ merged.append(buffer)
+ buffer = piece
+ if buffer and self._token_count(buffer) >= self.min_tokens:
+ merged.append(buffer)
+ buffer = ""
+ if buffer:
+ merged.append(buffer)
+ return merged
+
class RecursiveChunker(_BaseStrategyChunker):
"""Split on a separator hierarchy, capping at ``max_tokens``.
@@ -95,22 +117,7 @@ class RecursiveChunker(_BaseStrategyChunker):
def _merge(self, fragments: List[str]) -> List[str]:
"""Merge small fragments up to ``max_tokens`` to clear ``min_tokens``."""
- merged: List[str] = []
- buffer = ""
- for frag in fragments:
- candidate = buffer + frag if buffer else frag
- if self._token_count(candidate) <= self.max_tokens:
- buffer = candidate
- else:
- if buffer:
- merged.append(buffer)
- buffer = frag
- if buffer and self._token_count(buffer) >= self.min_tokens:
- merged.append(buffer)
- buffer = ""
- if buffer:
- merged.append(buffer)
- return merged
+ return self._merge_to_min(fragments, "")
def chunk(self, documents: List[Document]) -> List[Document]:
processed: List[Document] = []
@@ -210,9 +217,108 @@ class ParentChildChunker(_BaseStrategyChunker):
return processed
+class SemanticChunker(_BaseStrategyChunker):
+ """Group adjacent sentences by embedding similarity into coherent chunks.
+
+ Sentences are embedded in one batched call and split where the cosine
+ distance between consecutive sentences crosses a high percentile, so
+ chunk boundaries fall at topic shifts. Chunks are then token-capped at
+ ``max_tokens`` and merged up to clear ``min_tokens``. Any failure (too
+ few sentences, embedding error, degenerate distances) falls back to
+ ``RecursiveChunker`` so ingest never crashes.
+ """
+
+ _SENTENCE = re.compile(r"(?<=[.!?])\s+")
+ _PERCENTILE = 95.0
+
+ def _split_sentences(self, text: str) -> List[str]:
+ return [s for s in (p.strip() for p in self._SENTENCE.split(text)) if s]
+
+ def _fallback(self, documents: List[Document]) -> List[Document]:
+ recursive = RecursiveChunker(
+ chunking_strategy=self.chunking_strategy,
+ max_tokens=self.max_tokens,
+ min_tokens=self.min_tokens,
+ duplicate_headers=self.duplicate_headers,
+ )
+ return recursive.chunk(documents)
+
+ def _breakpoints(self, embeddings) -> set:
+ """Indices after which a new chunk starts, from high-distance gaps."""
+ import numpy as np
+
+ matrix = np.asarray(embeddings, dtype=np.float64)
+ if matrix.ndim != 2 or matrix.shape[0] < 2:
+ return set()
+ norms = np.linalg.norm(matrix, axis=1)
+ norms[norms == 0] = 1.0
+ unit = matrix / norms[:, None]
+ sims = np.sum(unit[:-1] * unit[1:], axis=1)
+ distances = 1.0 - sims
+ if not np.all(np.isfinite(distances)):
+ raise ValueError("non-finite semantic distances")
+ if float(distances.max()) <= float(distances.min()):
+ return set()
+ threshold = np.percentile(distances, self._PERCENTILE)
+ return {int(i) for i in np.where(distances >= threshold)[0]}
+
+ def _group(self, sentences: List[str], breakpoints: set) -> List[str]:
+ groups: List[str] = []
+ current: List[str] = []
+ for idx, sentence in enumerate(sentences):
+ current.append(sentence)
+ if idx in breakpoints:
+ groups.append(" ".join(current))
+ current = []
+ if current:
+ groups.append(" ".join(current))
+ return groups
+
+ def _enforce_tokens(self, groups: List[str]) -> List[str]:
+ """Hard-split groups over ``max_tokens`` then merge to clear min."""
+ capped: List[str] = []
+ for group in groups:
+ if self._token_count(group) <= self.max_tokens:
+ capped.append(group)
+ else:
+ capped.extend(p for p in self._split_by_tokens(group) if p.strip())
+ return self._merge_to_min(capped, " ")
+
+ def _chunk_text(self, text: str) -> List[str]:
+ sentences = self._split_sentences(text)
+ if len(sentences) < 2:
+ raise ValueError("too few sentences for semantic chunking")
+ from application.vectorstore.base import EmbeddingsSingleton
+ from application.core.settings import settings
+
+ embeddings = EmbeddingsSingleton.get_instance(
+ settings.EMBEDDINGS_NAME
+ ).embed_documents(sentences)
+ breakpoints = self._breakpoints(embeddings)
+ groups = self._group(sentences, breakpoints)
+ return [g for g in self._enforce_tokens(groups) if g.strip()]
+
+ def chunk(self, documents: List[Document]) -> List[Document]:
+ processed: List[Document] = []
+ for doc in documents:
+ try:
+ texts = self._chunk_text(doc.text)
+ except Exception as exc:
+ logger.warning(
+ "Semantic chunking failed (%s); falling back to recursive.",
+ exc,
+ )
+ processed.extend(self._fallback([doc]))
+ continue
+ for idx, text in enumerate(texts):
+ processed.append(self._emit(doc, idx, text))
+ return processed
+
+
ChunkerCreator.register("recursive", RecursiveChunker)
ChunkerCreator.register("markdown", MarkdownChunker)
ChunkerCreator.register("parent_child", ParentChildChunker)
+ChunkerCreator.register("semantic", SemanticChunker)
# Reuse the classic Chunker reference so this module can be the single import
# that pulls every strategy into the registry.
@@ -220,5 +326,6 @@ __all__ = [
"RecursiveChunker",
"MarkdownChunker",
"ParentChildChunker",
+ "SemanticChunker",
"Chunker",
]
diff --git a/application/requirements.txt b/application/requirements.txt
index 5414df03..822259c6 100644
--- a/application/requirements.txt
+++ b/application/requirements.txt
@@ -69,6 +69,7 @@ pandas==3.0.2
openpyxl==3.1.5
pathable==0.5.0
pdf2image>=1.17.0
+pgvector>=0.2,<1
pillow
portalocker>=2.7.0,<4.0.0
prompt-toolkit==3.0.52
diff --git a/application/retriever/classic_rag.py b/application/retriever/classic_rag.py
index a95e7750..154fc7b0 100644
--- a/application/retriever/classic_rag.py
+++ b/application/retriever/classic_rag.py
@@ -144,6 +144,21 @@ class ClassicRAG(BaseRetriever):
logging.error(f"Error rephrasing query: {e}", exc_info=True)
return self.original_question
+ def _fetch_candidates(self, docsearch, question, src_k, score_threshold):
+ """Fetch candidate hits for one vector store (vector search).
+
+ Subclasses override this to change candidate sourcing (e.g. RRF fusion)
+ while inheriting the surrounding per-source resolution and budgeting.
+ """
+ # ``score_threshold`` is honoured by pgvector/mongodb and safely ignored
+ # by stores whose ``search`` swallows kwargs. The candidate count is
+ # clamped to a ceiling to bound memory/latency.
+ k = min(max(src_k * 2, 20), 500)
+ search_kwargs = {"k": k}
+ if score_threshold is not None:
+ search_kwargs["score_threshold"] = score_threshold
+ return docsearch.search(question, **search_kwargs)
+
def _get_data(self):
if self.chunks == 0 or not self.vectorstores:
logging.info(
@@ -193,12 +208,9 @@ class ClassicRAG(BaseRetriever):
docsearch = VectorCreator.create_vectorstore(
settings.VECTOR_STORE, vectorstore_id, settings.EMBEDDINGS_KEY
)
- # ``score_threshold`` is honoured by pgvector/mongodb and
- # safely ignored by stores whose ``search`` swallows kwargs.
- search_kwargs = {"k": max(src_k * 2, 20)}
- if score_threshold is not None:
- search_kwargs["score_threshold"] = score_threshold
- docs_temp = docsearch.search(question, **search_kwargs)
+ docs_temp = self._fetch_candidates(
+ docsearch, question, src_k, score_threshold
+ )
for doc in docs_temp:
if cumulative_tokens >= token_budget:
diff --git a/application/retriever/dispatcher.py b/application/retriever/dispatcher.py
index f995aaa1..38ebd325 100644
--- a/application/retriever/dispatcher.py
+++ b/application/retriever/dispatcher.py
@@ -32,6 +32,11 @@ logger = logging.getLogger(__name__)
# for the all-classic case.
_CLASSIC_KEYS = frozenset({"classic", "default"})
+# Retriever keys that subclass ClassicRAG and accept ``defer_rephrase``, so the
+# eager ctor rephrase can be skipped and computed lazily per-source. Unknown
+# keys are excluded: a future non-ClassicRAG retriever may not accept the kwarg.
+_DEFERRABLE_KEYS = _CLASSIC_KEYS | {"hybrid"}
+
# Fields that, when left at their defaults, mean the source did not opt into any
# per-source retrieval override — so it can flow through the global ClassicRAG
# path unchanged (byte-identical parity).
@@ -220,7 +225,7 @@ class Dispatcher(BaseRetriever):
kwargs["chunks"] = max(self.chunks, candidate_k or 0)
# With per-source configs the rephrase decision is per-source, so defer
# the eager rephrase to let a rephrase_query=False source skip the call.
- if group["retrievals"] and retriever_key in _CLASSIC_KEYS:
+ if group["retrievals"] and retriever_key in _DEFERRABLE_KEYS:
kwargs["defer_rephrase"] = True
retriever = RetrieverCreator.create_retriever(retriever_key, **kwargs)
# Hand the per-source retrieval configs to the classic retriever so it
diff --git a/application/retriever/hybrid_rag.py b/application/retriever/hybrid_rag.py
new file mode 100644
index 00000000..b789ddc5
--- /dev/null
+++ b/application/retriever/hybrid_rag.py
@@ -0,0 +1,62 @@
+"""Hybrid retriever fusing vector and keyword search via RRF.
+
+Subclasses :class:`ClassicRAG` so the Dispatcher builds it with identical
+ctor kwargs and it inherits rephrase + token-budgeting. Only the per-source
+fetch is overridden: for each vector store it pulls vector hits and keyword
+hits, then fuses them with Reciprocal Rank Fusion. Stores without keyword
+support (``keyword_search`` returns ``[]``) reduce to exact vector-only
+behaviour.
+"""
+
+from application.retriever.classic_rag import ClassicRAG
+
+RRF_K = 60
+
+
+def _doc_key(doc):
+ """Stable identity for a hit so the same chunk fuses across both lists."""
+ if hasattr(doc, "page_content") and hasattr(doc, "metadata"):
+ content = doc.page_content
+ metadata = doc.metadata or {}
+ else:
+ content = doc.get("text", doc.get("page_content", ""))
+ metadata = doc.get("metadata") or {}
+ source = metadata.get("source", "")
+ return (source, content)
+
+
+def reciprocal_rank_fusion(vector_hits, keyword_hits, k=RRF_K):
+ """Fuse two ranked hit lists into one by Reciprocal Rank Fusion.
+
+ Each list contributes ``1 / (k + rank)`` per document (rank 0-based);
+ documents are returned ordered by summed score, highest first. A document
+ present in only one list is ranked solely on that list's contribution, so
+ an empty ``keyword_hits`` yields exactly the vector ordering.
+ """
+ scores = {}
+ docs = {}
+ for hits in (vector_hits, keyword_hits):
+ for rank, doc in enumerate(hits):
+ key = _doc_key(doc)
+ scores[key] = scores.get(key, 0.0) + 1.0 / (k + rank)
+ if key not in docs:
+ docs[key] = doc
+ ordered = sorted(docs.keys(), key=lambda key: scores[key], reverse=True)
+ return [docs[key] for key in ordered]
+
+
+class HybridRetriever(ClassicRAG):
+ """ClassicRAG variant that fuses vector + keyword search with RRF."""
+
+ def _fetch_candidates(self, docsearch, question, src_k, score_threshold):
+ """Return RRF-fused vector+keyword hits for one vector store.
+
+ Inherits the per-source resolution and budgeting from
+ :meth:`ClassicRAG._get_data`; only candidate sourcing differs.
+ RRF scores are not cosine similarities, so ``score_threshold`` is
+ intentionally not applied to the fused list.
+ """
+ candidate_k = min(max(src_k * 2, 20), 500)
+ vector_hits = docsearch.search(question, k=candidate_k)
+ keyword_hits = docsearch.keyword_search(question, k=candidate_k)
+ return reciprocal_rank_fusion(vector_hits, keyword_hits)
diff --git a/application/retriever/retriever_creator.py b/application/retriever/retriever_creator.py
index 2b507bf3..43d75035 100644
--- a/application/retriever/retriever_creator.py
+++ b/application/retriever/retriever_creator.py
@@ -1,10 +1,12 @@
from application.retriever.classic_rag import ClassicRAG
+from application.retriever.hybrid_rag import HybridRetriever
class RetrieverCreator:
retrievers = {
"classic": ClassicRAG,
"default": ClassicRAG,
+ "hybrid": HybridRetriever,
}
@classmethod
diff --git a/application/storage/db/source_config.py b/application/storage/db/source_config.py
index fdf648dd..540073e7 100644
--- a/application/storage/db/source_config.py
+++ b/application/storage/db/source_config.py
@@ -73,6 +73,15 @@ class RetrievalConfig(BaseModel):
reranker: Optional[dict] = None # reserved: future cross-encoder/LLM reorder
prescreen: Optional[dict] = None # None = off; else PreScreenConfig dict (D12)
+ @field_validator("chunks")
+ @classmethod
+ def _bounded_chunks(cls, value: int) -> int:
+ if value < 1:
+ raise ValueError("must be >= 1")
+ if value > 500:
+ raise ValueError("must be <= 500")
+ return value
+
@model_validator(mode="after")
def _validate_prescreen(self) -> "RetrievalConfig":
"""Validate ``prescreen`` through ``PreScreenConfig`` when present.
diff --git a/application/vectorstore/base.py b/application/vectorstore/base.py
index b3cf4da4..d61177a8 100644
--- a/application/vectorstore/base.py
+++ b/application/vectorstore/base.py
@@ -136,6 +136,14 @@ class BaseVectorStore(ABC):
"""Search for similar documents/chunks in the vectorstore"""
pass
+ def keyword_search(self, question, k=10):
+ """Keyword/full-text search.
+
+ Default returns no results so hybrid retrieval degrades to vector-only
+ on stores without keyword support. Override in stores that support it.
+ """
+ return []
+
@abstractmethod
def add_texts(self, texts, metadatas=None, *args, **kwargs):
"""Add texts with their embeddings to the vectorstore"""
diff --git a/application/vectorstore/pgvector.py b/application/vectorstore/pgvector.py
index 43b1585f..78925cbb 100644
--- a/application/vectorstore/pgvector.py
+++ b/application/vectorstore/pgvector.py
@@ -102,11 +102,18 @@ class PGVectorStore(BaseVectorStore):
# Create index for source_id filtering
source_index_query = f"""
- CREATE INDEX IF NOT EXISTS {self._table_name}_source_id_idx
+ CREATE INDEX IF NOT EXISTS {self._table_name}_source_id_idx
ON {self._table_name} (source_id);
"""
cursor.execute(source_index_query)
-
+
+ # Functional GIN index backing keyword_search full-text queries.
+ fts_index_query = f"""
+ CREATE INDEX IF NOT EXISTS {self._table_name}_text_fts_idx
+ ON {self._table_name} USING gin(to_tsvector('english', {self._text_column}));
+ """
+ cursor.execute(fts_index_query)
+
conn.commit()
except Exception as e:
conn.rollback()
@@ -170,6 +177,48 @@ class PGVectorStore(BaseVectorStore):
finally:
cursor.close()
+ def keyword_search(self, question: str, k: int = 10) -> List[Document]:
+ """Full-text keyword search using Postgres ``websearch_to_tsquery``.
+
+ Returns the same ``Document`` shape as :meth:`search`. The question is
+ bound as a query parameter (never interpolated) to prevent injection.
+ """
+ conn = self._get_connection()
+ cursor = conn.cursor()
+
+ try:
+ keyword_query = f"""
+ SELECT {self._text_column}, {self._metadata_column},
+ ts_rank(
+ to_tsvector('english', {self._text_column}),
+ websearch_to_tsquery('english', %s)
+ ) AS rank
+ FROM {self._table_name}
+ WHERE source_id = %s
+ AND to_tsvector('english', {self._text_column})
+ @@ websearch_to_tsquery('english', %s)
+ ORDER BY rank DESC
+ LIMIT %s;
+ """
+
+ cursor.execute(
+ keyword_query, (question, self._source_id, question, k)
+ )
+ results = cursor.fetchall()
+
+ documents = []
+ for text, metadata, _rank in results:
+ metadata = metadata or {}
+ documents.append(Document(page_content=text, metadata=metadata))
+
+ return documents
+
+ except Exception as e:
+ logging.error(f"Error in keyword search: {e}", exc_info=True)
+ return []
+ finally:
+ cursor.close()
+
def add_texts(
self,
texts: List[str],
diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json
index 3bd8400a..4f5330e7 100644
--- a/frontend/src/locale/en.json
+++ b/frontend/src/locale/en.json
@@ -145,8 +145,10 @@
"rephraseQuery": "Rephrase query before search",
"retriever": "Retriever",
"retrievers": {
- "classic": "Classic"
+ "classic": "Classic",
+ "hybrid": "Hybrid"
},
+ "retrieverHint": "Hybrid (keyword + vector) search is supported on pgvector; other vector stores fall back to vector-only.",
"exposure": "Search exposure",
"exposures": {
"prefetch": "Pre-fetch into context",
@@ -172,7 +174,8 @@
"classic_chunk": "Classic",
"recursive": "Recursive",
"markdown": "Markdown",
- "parent_child": "Parent / child"
+ "parent_child": "Parent / child",
+ "semantic": "Semantic"
},
"maxTokens": "Max tokens per chunk",
"minTokens": "Min tokens per chunk",
diff --git a/frontend/src/models/misc.ts b/frontend/src/models/misc.ts
index 004e397a..7b821f2b 100644
--- a/frontend/src/models/misc.ts
+++ b/frontend/src/models/misc.ts
@@ -11,7 +11,8 @@ export type ChunkingStrategy =
| 'classic_chunk'
| 'recursive'
| 'markdown'
- | 'parent_child';
+ | 'parent_child'
+ | 'semantic';
export type RetrievalExposure = 'prefetch' | 'agentic_tool';
diff --git a/frontend/src/settings/components/RetrievalOptions.tsx b/frontend/src/settings/components/RetrievalOptions.tsx
index 9552fff1..fb2a82a6 100644
--- a/frontend/src/settings/components/RetrievalOptions.tsx
+++ b/frontend/src/settings/components/RetrievalOptions.tsx
@@ -83,8 +83,11 @@ const CHUNKING_STRATEGIES: ChunkingStrategy[] = [
'recursive',
'markdown',
'parent_child',
+ 'semantic',
];
+const RETRIEVERS = ['classic', 'hybrid'];
+
/**
* Group eyebrow: an uppercase, tracked title with a normal-weight muted ` · tag`
* suffix. Reads as more prominent than the individual field labels below it.
@@ -302,6 +305,34 @@ export default function RetrievalOptions({