mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-20 02:16:39 +00:00
feat: parser improvements and fixes
This commit is contained in:
1 parent
a70362a5f3
commit
0b36257202
55 files changed
+7712
-442
No files matched your search
@@ -6,8 +6,12 @@ from application.vectorstore.base import (
|
||||
BaseVectorStore,
|
||||
EmbeddingsSingleton,
|
||||
RemoteEmbeddings,
|
||||
get_embeddings,
|
||||
)
|
||||
|
||||
HF_MPNET = "huggingface_sentence-transformers/all-mpnet-base-v2"
|
||||
LOCAL_MPNET = "/app/models/all-mpnet-base-v2"
|
||||
|
||||
|
||||
# --- RemoteEmbeddings ---
|
||||
|
||||
@@ -254,6 +258,46 @@ class TestEmbeddingsSingleton:
|
||||
assert result.headers["Authorization"] == "Bearer sk-from-settings"
|
||||
|
||||
|
||||
@patch("application.vectorstore.base.settings")
|
||||
@patch("application.vectorstore.base._get_embeddings_wrapper")
|
||||
def test_get_instance_hf_ignores_positional_key(
|
||||
self, mock_get_wrapper, mock_settings
|
||||
):
|
||||
"""A stray key must not reach the zero-arg HuggingFace factory.
|
||||
|
||||
The factories are ``lambda: EmbeddingsWrapper(...)``, so a caller that
|
||||
passed ``settings.EMBEDDINGS_KEY`` positionally used to blow up with
|
||||
``TypeError: <lambda>() takes 0 positional arguments``.
|
||||
"""
|
||||
mock_settings.EMBEDDINGS_BASE_URL = None
|
||||
mock_wrapper_cls = Mock()
|
||||
mock_instance = Mock()
|
||||
mock_wrapper_cls.return_value = mock_instance
|
||||
mock_get_wrapper.return_value = mock_wrapper_cls
|
||||
|
||||
result = EmbeddingsSingleton.get_instance(HF_MPNET, None)
|
||||
|
||||
assert result is mock_instance
|
||||
mock_wrapper_cls.assert_called_once_with(
|
||||
"sentence-transformers/all-mpnet-base-v2"
|
||||
)
|
||||
|
||||
@patch("application.vectorstore.base.settings")
|
||||
@patch("application.vectorstore.base._get_embeddings_wrapper")
|
||||
def test_get_instance_hf_ignores_keyword_args(
|
||||
self, mock_get_wrapper, mock_settings
|
||||
):
|
||||
mock_settings.EMBEDDINGS_BASE_URL = None
|
||||
mock_wrapper_cls = Mock()
|
||||
mock_get_wrapper.return_value = mock_wrapper_cls
|
||||
|
||||
EmbeddingsSingleton.get_instance(HF_MPNET, openai_api_key="sk-nope")
|
||||
|
||||
mock_wrapper_cls.assert_called_once_with(
|
||||
"sentence-transformers/all-mpnet-base-v2"
|
||||
)
|
||||
|
||||
|
||||
# --- BaseVectorStore ---
|
||||
|
||||
|
||||
@@ -415,3 +459,159 @@ class TestSearchWithScoresDefault:
|
||||
return []
|
||||
|
||||
assert _Store().search_with_scores("q") == []
|
||||
|
||||
|
||||
# --- get_embeddings (the single resolver) ---
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestGetEmbeddingsResolver:
|
||||
"""``get_embeddings`` is the one entry point every caller must use.
|
||||
|
||||
Calling ``EmbeddingsSingleton.get_instance`` directly reproduces neither the
|
||||
bundled local-model path nor the OpenAI/Azure key handling.
|
||||
"""
|
||||
|
||||
def setup_method(self):
|
||||
EmbeddingsSingleton._instances = {}
|
||||
|
||||
@patch("application.vectorstore.base.settings")
|
||||
@patch("application.vectorstore.base._get_embeddings_wrapper")
|
||||
@patch("os.path.exists", return_value=False)
|
||||
def test_defaults_from_settings_do_not_raise(
|
||||
self, _mock_exists, mock_get_wrapper, mock_settings
|
||||
):
|
||||
"""The default config (HF mpnet name, no key) must resolve, not crash."""
|
||||
mock_settings.EMBEDDINGS_BASE_URL = None
|
||||
mock_settings.EMBEDDINGS_NAME = HF_MPNET
|
||||
mock_settings.EMBEDDINGS_KEY = None
|
||||
mock_wrapper_cls = Mock()
|
||||
mock_instance = Mock()
|
||||
mock_wrapper_cls.return_value = mock_instance
|
||||
mock_get_wrapper.return_value = mock_wrapper_cls
|
||||
|
||||
result = get_embeddings()
|
||||
|
||||
assert result is mock_instance
|
||||
assert set(EmbeddingsSingleton._instances) == {HF_MPNET}
|
||||
|
||||
@patch("application.vectorstore.base.settings")
|
||||
@patch("application.vectorstore.base._get_embeddings_wrapper")
|
||||
@patch("os.path.exists", return_value=False)
|
||||
def test_shares_cache_entry_with_vectorstore_helper(
|
||||
self, _mock_exists, mock_get_wrapper, mock_settings
|
||||
):
|
||||
"""Same object, same cache key as the vector stores get — one model."""
|
||||
mock_settings.EMBEDDINGS_BASE_URL = None
|
||||
mock_settings.EMBEDDINGS_NAME = HF_MPNET
|
||||
mock_settings.EMBEDDINGS_KEY = None
|
||||
mock_wrapper_cls = Mock()
|
||||
mock_wrapper_cls.return_value = Mock()
|
||||
mock_get_wrapper.return_value = mock_wrapper_cls
|
||||
|
||||
store_result = ConcreteVectorStore()._get_embeddings(HF_MPNET, None)
|
||||
resolver_result = get_embeddings()
|
||||
|
||||
assert resolver_result is store_result
|
||||
assert set(EmbeddingsSingleton._instances) == {HF_MPNET}
|
||||
mock_wrapper_cls.assert_called_once()
|
||||
|
||||
@patch("application.vectorstore.base.settings")
|
||||
@patch("application.vectorstore.base._get_embeddings_wrapper")
|
||||
@patch("os.path.exists")
|
||||
def test_uses_local_model_path_and_caches_it(
|
||||
self, mock_exists, mock_get_wrapper, mock_settings
|
||||
):
|
||||
"""With the bundled model present, the instance is keyed by its path.
|
||||
|
||||
A second call must not load a second copy of the model.
|
||||
"""
|
||||
mock_settings.EMBEDDINGS_BASE_URL = None
|
||||
mock_settings.EMBEDDINGS_NAME = HF_MPNET
|
||||
mock_settings.EMBEDDINGS_KEY = None
|
||||
mock_exists.side_effect = lambda path: path == LOCAL_MPNET
|
||||
mock_wrapper_cls = Mock()
|
||||
mock_wrapper_cls.return_value = Mock()
|
||||
mock_get_wrapper.return_value = mock_wrapper_cls
|
||||
|
||||
first = get_embeddings()
|
||||
second = get_embeddings()
|
||||
|
||||
assert first is second
|
||||
assert set(EmbeddingsSingleton._instances) == {LOCAL_MPNET}
|
||||
mock_wrapper_cls.assert_called_once_with(LOCAL_MPNET)
|
||||
|
||||
@patch("application.vectorstore.base.settings")
|
||||
def test_remote_when_base_url_configured(self, mock_settings):
|
||||
mock_settings.EMBEDDINGS_BASE_URL = "http://remote:8080"
|
||||
mock_settings.EMBEDDINGS_NAME = HF_MPNET
|
||||
mock_settings.EMBEDDINGS_KEY = "sk-remote"
|
||||
|
||||
result = get_embeddings()
|
||||
|
||||
assert isinstance(result, RemoteEmbeddings)
|
||||
assert result.api_url == "http://remote:8080"
|
||||
assert result.model_name == HF_MPNET
|
||||
assert result.headers["Authorization"] == "Bearer sk-remote"
|
||||
|
||||
@patch("application.vectorstore.base.settings")
|
||||
@patch("application.vectorstore.base.EmbeddingsSingleton.get_instance")
|
||||
def test_openai_passes_key(self, mock_get_instance, mock_settings):
|
||||
mock_settings.EMBEDDINGS_BASE_URL = None
|
||||
mock_settings.OPENAI_API_BASE = None
|
||||
mock_settings.OPENAI_API_VERSION = None
|
||||
mock_settings.AZURE_DEPLOYMENT_NAME = None
|
||||
mock_settings.EMBEDDINGS_NAME = "openai_text-embedding-ada-002"
|
||||
mock_settings.EMBEDDINGS_KEY = "sk-from-settings"
|
||||
|
||||
get_embeddings()
|
||||
|
||||
mock_get_instance.assert_called_once_with(
|
||||
"openai_text-embedding-ada-002", openai_api_key="sk-from-settings"
|
||||
)
|
||||
|
||||
@patch("application.vectorstore.base.settings")
|
||||
@patch("application.vectorstore.base.EmbeddingsSingleton.get_instance")
|
||||
def test_openai_azure_uses_deployment_name(
|
||||
self, mock_get_instance, mock_settings
|
||||
):
|
||||
mock_settings.EMBEDDINGS_BASE_URL = None
|
||||
mock_settings.OPENAI_API_BASE = "https://azure.openai.com"
|
||||
mock_settings.OPENAI_API_VERSION = "2023-05-15"
|
||||
mock_settings.AZURE_DEPLOYMENT_NAME = "deploy"
|
||||
mock_settings.AZURE_EMBEDDINGS_DEPLOYMENT_NAME = "embed-deploy"
|
||||
mock_settings.EMBEDDINGS_NAME = "openai_text-embedding-ada-002"
|
||||
mock_settings.EMBEDDINGS_KEY = "sk-key"
|
||||
|
||||
get_embeddings()
|
||||
|
||||
mock_get_instance.assert_called_once_with(
|
||||
"openai_text-embedding-ada-002", model="embed-deploy"
|
||||
)
|
||||
|
||||
@patch("application.vectorstore.base.settings")
|
||||
@patch("application.vectorstore.base.EmbeddingsSingleton.get_instance")
|
||||
def test_explicit_arguments_win_over_settings(
|
||||
self, mock_get_instance, mock_settings
|
||||
):
|
||||
mock_settings.EMBEDDINGS_BASE_URL = None
|
||||
mock_settings.EMBEDDINGS_NAME = HF_MPNET
|
||||
mock_settings.EMBEDDINGS_KEY = "sk-from-settings"
|
||||
|
||||
get_embeddings("some_custom_embedding", "sk-explicit")
|
||||
|
||||
mock_get_instance.assert_called_once_with("some_custom_embedding")
|
||||
|
||||
@patch("application.vectorstore.base.settings")
|
||||
@patch("application.vectorstore.base.get_embeddings")
|
||||
def test_vectorstore_helper_delegates_to_resolver(
|
||||
self, mock_resolver, _mock_settings
|
||||
):
|
||||
"""``BaseVectorStore._get_embeddings`` is a thin delegate now."""
|
||||
sentinel = Mock()
|
||||
mock_resolver.return_value = sentinel
|
||||
|
||||
result = ConcreteVectorStore()._get_embeddings("a-name", "a-key")
|
||||
|
||||
assert result is sentinel
|
||||
mock_resolver.assert_called_once_with("a-name", "a-key")
|
||||
@@ -31,13 +31,17 @@ def _make_store(
|
||||
|
||||
from application.vectorstore.pgvector import PGVectorStore
|
||||
|
||||
# Patch _ensure_table_exists to avoid DB calls during init
|
||||
with patch.object(PGVectorStore, "_ensure_table_exists"):
|
||||
store = PGVectorStore(
|
||||
source_id=source_id,
|
||||
embeddings_key=embeddings_key,
|
||||
connection_string=connection_string,
|
||||
)
|
||||
store = PGVectorStore(
|
||||
source_id=source_id,
|
||||
embeddings_key=embeddings_key,
|
||||
connection_string=connection_string,
|
||||
)
|
||||
# Boot owns the schema, so these unit stores start already ensured;
|
||||
# the write-path safety net has its own tests.
|
||||
store._schema_ensured = True
|
||||
# Legacy direct-connection mode: pooling would build a real
|
||||
# psycopg_pool against the fake DSN. The pooled path has its own tests.
|
||||
store._pool_max_size = 0
|
||||
# Provide a mock connection
|
||||
mock_conn = MagicMock()
|
||||
mock_cursor = MagicMock()
|
||||
@@ -438,3 +442,47 @@ class TestPGVectorSearchWithScores:
|
||||
mock_cursor.execute.side_effect = Exception("connection lost")
|
||||
|
||||
assert store.search_with_scores("query") == []
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestPGVectorPrecomputedQueryVector:
|
||||
"""A caller that already embedded the query must not pay for it again.
|
||||
|
||||
Multi-source retrieval embeds the query once and hands the same vector to
|
||||
every source's store; without this the query is embedded once per source.
|
||||
"""
|
||||
|
||||
def test_search_with_scores_skips_embedding(self):
|
||||
store, _, mock_cursor, mock_emb = _make_store()
|
||||
mock_cursor.fetchall.return_value = [("hello world", {}, 0.1)]
|
||||
|
||||
results = store.search_with_scores("query", k=2, query_vector=[0.9, 0.8, 0.7])
|
||||
|
||||
mock_emb.embed_query.assert_not_called()
|
||||
assert [doc.page_content for doc, _ in results] == ["hello world"]
|
||||
|
||||
def test_search_skips_embedding(self):
|
||||
store, _, mock_cursor, mock_emb = _make_store()
|
||||
mock_cursor.fetchall.return_value = [("hello world", {}, 0.1)]
|
||||
|
||||
results = store.search("query", k=2, query_vector=[0.9, 0.8, 0.7])
|
||||
|
||||
mock_emb.embed_query.assert_not_called()
|
||||
assert [doc.page_content for doc in results] == ["hello world"]
|
||||
|
||||
def test_supplied_vector_is_what_the_query_binds(self):
|
||||
store, _, mock_cursor, _ = _make_store(source_id="src1")
|
||||
mock_cursor.fetchall.return_value = [("a", {}, 0.1)]
|
||||
|
||||
store.search_with_scores("query", k=1, query_vector=[0.9, 0.8, 0.7])
|
||||
|
||||
_sql, params = mock_cursor.execute.call_args_list[0].args
|
||||
assert params == ([0.9, 0.8, 0.7], "src1", [0.9, 0.8, 0.7], 1)
|
||||
|
||||
def test_none_vector_still_embeds(self):
|
||||
store, _, mock_cursor, mock_emb = _make_store()
|
||||
mock_cursor.fetchall.return_value = []
|
||||
|
||||
store.search_with_scores("query", k=2, query_vector=None)
|
||||
|
||||
mock_emb.embed_query.assert_called_once_with("query")
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Live pgvector end-to-end: boot-owned schema, pure-read queries, pooling.
|
||||
|
||||
Runs against the ephemeral pytest-postgresql cluster (never the operator's dev
|
||||
database) with an 8-dimension stub embeddings model, so a real ``CREATE
|
||||
EXTENSION vector`` / ``vector(8)`` round trip is exercised in a few seconds.
|
||||
Skips when the cluster's Postgres has no pgvector build installed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from application.storage.db.bootstrap import ensure_vector_schema
|
||||
from application.vectorstore import pgvector as pgvector_module
|
||||
from application.vectorstore.pgvector import PGVectorStore
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
STUB_DIM = 8
|
||||
|
||||
|
||||
class _StubEmbeddings:
|
||||
"""Deterministic tiny embeddings; no model download, no network."""
|
||||
|
||||
dimension = STUB_DIM
|
||||
|
||||
def embed_documents(self, texts):
|
||||
return [self._vector(float(i + 1)) for i, _ in enumerate(texts)]
|
||||
|
||||
def embed_query(self, query):
|
||||
return self._vector(1.0)
|
||||
|
||||
@staticmethod
|
||||
def _vector(seed: float):
|
||||
vector = [0.0] * STUB_DIM
|
||||
vector[0] = seed
|
||||
return vector
|
||||
|
||||
|
||||
class _WideStubEmbeddings(_StubEmbeddings):
|
||||
dimension = 16
|
||||
|
||||
|
||||
def _dsn(info) -> str:
|
||||
password = f":{info.password}" if info.password else ""
|
||||
return (
|
||||
f"postgresql://{info.user}{password}@{info.host}:{info.port}/{info.dbname}"
|
||||
)
|
||||
|
||||
|
||||
def _regclass(conn, name: str):
|
||||
"""Look a relation up on ``conn``, in a transaction that starts fresh.
|
||||
|
||||
``to_regclass`` takes no lock, so a negative catalog-cache entry from an
|
||||
earlier lookup in the same transaction survives another backend's DDL. A
|
||||
new transaction accepts the invalidation.
|
||||
"""
|
||||
conn.rollback()
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute("SELECT to_regclass(%s);", (name,))
|
||||
return cursor.fetchone()[0]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _close_pools():
|
||||
"""Never leak a pool into another test; the DSN dies with the test DB."""
|
||||
yield
|
||||
for dsn, pool in list(pgvector_module._POOLS.items()):
|
||||
try:
|
||||
pool.close()
|
||||
except Exception:
|
||||
pass
|
||||
pgvector_module._POOLS.pop(dsn, None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_dsn(postgresql, monkeypatch):
|
||||
"""DSN of an empty ephemeral DB, with pgvector proven installable."""
|
||||
try:
|
||||
with postgresql.cursor() as cursor:
|
||||
cursor.execute("CREATE EXTENSION vector;")
|
||||
postgresql.rollback() # leave the database pristine for the hook
|
||||
except Exception as exc:
|
||||
postgresql.rollback()
|
||||
pytest.skip(f"pgvector extension unavailable: {exc}")
|
||||
|
||||
dsn = _dsn(postgresql.info)
|
||||
from application.core import settings as settings_module
|
||||
|
||||
settings = settings_module.settings
|
||||
monkeypatch.setattr(settings, "VECTOR_STORE", "pgvector", raising=False)
|
||||
monkeypatch.setattr(
|
||||
settings, "PGVECTOR_CONNECTION_STRING", dsn, raising=False
|
||||
)
|
||||
monkeypatch.setattr(settings, "GRAPHRAG_ENABLED", False, raising=False)
|
||||
monkeypatch.setattr(settings, "PGVECTOR_IVFFLAT_PROBES", None, raising=False)
|
||||
monkeypatch.setattr(settings, "PGVECTOR_POOL_MAX_SIZE", 4, raising=False)
|
||||
return dsn
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_embeddings():
|
||||
stub = _StubEmbeddings()
|
||||
with patch(
|
||||
"application.vectorstore.base.get_embeddings", return_value=stub
|
||||
), patch(
|
||||
"application.vectorstore.base.BaseVectorStore._get_embeddings",
|
||||
return_value=stub,
|
||||
):
|
||||
yield stub
|
||||
|
||||
|
||||
def _store(dsn, source_id="live-source"):
|
||||
return PGVectorStore(source_id=source_id, connection_string=dsn)
|
||||
|
||||
|
||||
class TestBootHookCreatesTheSchema:
|
||||
def test_creates_the_table_and_indexes_and_is_idempotent(
|
||||
self, live_dsn, postgresql, stub_embeddings
|
||||
):
|
||||
ensure_vector_schema()
|
||||
ensure_vector_schema() # second boot must be a no-op
|
||||
|
||||
assert _regclass(postgresql, "documents") is not None
|
||||
with postgresql.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"SELECT indexname FROM pg_indexes WHERE tablename = 'documents';"
|
||||
)
|
||||
indexes = {row[0] for row in cursor.fetchall()}
|
||||
assert "documents_source_id_idx" in indexes
|
||||
assert "documents_text_fts_idx" in indexes
|
||||
assert PGVectorStore.table_dimension(postgresql) == STUB_DIM
|
||||
|
||||
def test_mismatched_model_width_fails_loudly(self, live_dsn, stub_embeddings):
|
||||
ensure_vector_schema()
|
||||
|
||||
wide = _WideStubEmbeddings()
|
||||
with patch(
|
||||
"application.vectorstore.base.get_embeddings", return_value=wide
|
||||
):
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
ensure_vector_schema()
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert f"vector({STUB_DIM})" in message
|
||||
assert "16-dim" in message
|
||||
|
||||
def test_creates_graph_tables_when_graphrag_is_enabled(
|
||||
self, live_dsn, postgresql, stub_embeddings, monkeypatch
|
||||
):
|
||||
from application.core import settings as settings_module
|
||||
|
||||
monkeypatch.setattr(
|
||||
settings_module.settings, "GRAPHRAG_ENABLED", True, raising=False
|
||||
)
|
||||
|
||||
ensure_vector_schema()
|
||||
|
||||
for table in (
|
||||
"graph_nodes",
|
||||
"graph_edges",
|
||||
"graph_node_chunks",
|
||||
"graph_ingest_progress",
|
||||
):
|
||||
assert _regclass(postgresql, table) is not None
|
||||
|
||||
|
||||
class TestStoreRoundTrip:
|
||||
def test_construction_is_free_and_writes_then_reads_survive(
|
||||
self, live_dsn, stub_embeddings
|
||||
):
|
||||
ensure_vector_schema()
|
||||
|
||||
store = _store(live_dsn)
|
||||
# Construction must not open a connection or build a pool: the
|
||||
# retriever makes one of these per source, per request.
|
||||
assert store._connection is None
|
||||
assert pgvector_module._POOLS == {}
|
||||
|
||||
ids = store.add_texts(
|
||||
["alpha chunk", "beta chunk"],
|
||||
[{"source": "a.txt"}, {"source": "b.txt"}],
|
||||
)
|
||||
assert len(ids) == 2
|
||||
|
||||
results = store.search_with_scores("alpha", k=2)
|
||||
assert [doc.page_content for doc, _ in results] == [
|
||||
"alpha chunk",
|
||||
"beta chunk",
|
||||
]
|
||||
assert results[0][1] == pytest.approx(1.0)
|
||||
|
||||
keyword_hits = store.keyword_search("alpha", k=5)
|
||||
assert [doc.page_content for doc in keyword_hits] == ["alpha chunk"]
|
||||
|
||||
# The read paths must not leave the pooled connection in a transaction.
|
||||
assert store._connection.info.transaction_status.name == "IDLE"
|
||||
store.close()
|
||||
assert store._connection is None
|
||||
|
||||
|
||||
class TestWritePathSafetyNet:
|
||||
"""A process that never ran the boot hook must still be able to ingest.
|
||||
|
||||
``register_vector`` raises when the ``vector`` type is absent, which is the
|
||||
state of a brand-new database — so the store has to tolerate that, create
|
||||
the extension, and re-register before it inserts a vector.
|
||||
"""
|
||||
|
||||
def test_first_write_bootstraps_a_brand_new_database(
|
||||
self, live_dsn, postgresql, stub_embeddings
|
||||
):
|
||||
assert _regclass(postgresql, "documents") is None
|
||||
|
||||
store = _store(live_dsn)
|
||||
store.add_texts(["alpha chunk"], [{"source": "a.txt"}])
|
||||
|
||||
|
||||
assert _regclass(postgresql, "documents") is not None
|
||||
assert [doc.page_content for doc in store.search("alpha", k=2)] == [
|
||||
"alpha chunk"
|
||||
]
|
||||
store.close()
|
||||
|
||||
|
||||
class TestPooling:
|
||||
def test_concurrent_stores_share_one_pool_and_return_connections(
|
||||
self, live_dsn, stub_embeddings
|
||||
):
|
||||
ensure_vector_schema()
|
||||
seed = _store(live_dsn)
|
||||
seed.add_texts(["alpha chunk"], [{"source": "a.txt"}])
|
||||
seed.close()
|
||||
|
||||
errors: list[Exception] = []
|
||||
hits: list[int] = []
|
||||
|
||||
def _search():
|
||||
store = _store(live_dsn)
|
||||
try:
|
||||
hits.append(len(store.search("alpha", k=2)))
|
||||
except Exception as exc: # surfaced below
|
||||
errors.append(exc)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
threads = [threading.Thread(target=_search) for _ in range(2)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join(timeout=20)
|
||||
|
||||
assert errors == []
|
||||
assert hits == [1, 1]
|
||||
assert list(pgvector_module._POOLS) == [live_dsn]
|
||||
stats = pgvector_module._POOLS[live_dsn].get_stats()
|
||||
assert stats["pool_size"] <= 4
|
||||
assert stats["pool_available"] == stats["pool_size"]
|
||||
@@ -109,10 +109,10 @@ class TestNoVectorIndexOnEmptyTable:
|
||||
sources with hundreds of chunks, silently.
|
||||
"""
|
||||
|
||||
def test_ensure_table_exists_creates_no_vector_index(self):
|
||||
def test_create_schema_creates_no_vector_index(self):
|
||||
import inspect
|
||||
|
||||
source = inspect.getsource(PGVectorStore._ensure_table_exists)
|
||||
source = inspect.getsource(PGVectorStore.create_schema)
|
||||
assert "USING ivfflat" not in source
|
||||
assert "USING hnsw" not in source
|
||||
# the non-vector indexes are still expected
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
"""Schema ownership and connection pooling for ``PGVectorStore``.
|
||||
|
||||
Constructing a store used to run six DDL statements plus a fresh unpooled
|
||||
connect, on every instantiation — and the retriever builds one store per source
|
||||
per request. Schema is now owned at boot (``ensure_vector_schema``); the query
|
||||
path is pure reads over a pooled connection, and the write path keeps a
|
||||
one-shot safety net for processes that never ran the boot hook.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from application.vectorstore import pgvector as pgvector_module
|
||||
|
||||
CONNECTION_STRING = "postgresql://user:pass@localhost/db"
|
||||
|
||||
|
||||
def _make_store(source_id="test-source", connection_string=CONNECTION_STRING):
|
||||
"""Build a store with every external dependency mocked, no DDL patching."""
|
||||
with patch(
|
||||
"application.vectorstore.base.BaseVectorStore._get_embeddings"
|
||||
) as mock_get_emb, patch(
|
||||
"application.vectorstore.pgvector.settings"
|
||||
) as mock_settings, patch.dict(
|
||||
"sys.modules",
|
||||
{
|
||||
"psycopg": MagicMock(),
|
||||
"pgvector": MagicMock(),
|
||||
"pgvector.psycopg": MagicMock(),
|
||||
},
|
||||
):
|
||||
mock_emb = Mock()
|
||||
mock_emb.embed_query = Mock(return_value=[0.1, 0.2, 0.3])
|
||||
mock_emb.embed_documents = Mock(return_value=[[0.1, 0.2, 0.3]])
|
||||
mock_emb.dimension = 768
|
||||
mock_get_emb.return_value = mock_emb
|
||||
mock_settings.EMBEDDINGS_NAME = "test_model"
|
||||
mock_settings.PGVECTOR_CONNECTION_STRING = connection_string
|
||||
|
||||
from application.vectorstore.pgvector import PGVectorStore
|
||||
|
||||
store = PGVectorStore(
|
||||
source_id=source_id,
|
||||
embeddings_key="key",
|
||||
connection_string=connection_string,
|
||||
)
|
||||
|
||||
mock_conn = MagicMock()
|
||||
mock_cursor = MagicMock()
|
||||
mock_conn.cursor.return_value = mock_cursor
|
||||
mock_conn.closed = False
|
||||
store._connection = mock_conn
|
||||
store._pooled = False
|
||||
store._ensure_table_exists = Mock()
|
||||
return store, mock_conn, mock_cursor, mock_emb
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestConstructionTouchesNothing:
|
||||
def test_init_opens_no_connection_and_runs_no_ddl(self):
|
||||
with patch(
|
||||
"application.vectorstore.base.BaseVectorStore._get_embeddings"
|
||||
) as mock_get_emb, patch(
|
||||
"application.vectorstore.pgvector.settings"
|
||||
) as mock_settings, patch.dict(
|
||||
"sys.modules",
|
||||
{
|
||||
"psycopg": MagicMock(),
|
||||
"pgvector": MagicMock(),
|
||||
"pgvector.psycopg": MagicMock(),
|
||||
},
|
||||
):
|
||||
mock_get_emb.return_value = Mock(dimension=768)
|
||||
mock_settings.EMBEDDINGS_NAME = "test_model"
|
||||
mock_settings.PGVECTOR_CONNECTION_STRING = CONNECTION_STRING
|
||||
|
||||
from application.vectorstore.pgvector import PGVectorStore
|
||||
|
||||
with patch.object(
|
||||
PGVectorStore, "_get_connection"
|
||||
) as mock_get_conn, patch.object(
|
||||
PGVectorStore, "_ensure_table_exists"
|
||||
) as mock_ensure:
|
||||
store = PGVectorStore(
|
||||
source_id="s1", connection_string=CONNECTION_STRING
|
||||
)
|
||||
|
||||
mock_get_conn.assert_not_called()
|
||||
mock_ensure.assert_not_called()
|
||||
assert store._schema_ensured is False
|
||||
|
||||
def test_pool_size_falls_back_when_setting_is_not_an_int(self):
|
||||
# The unit suite patches ``settings`` with a MagicMock; a MagicMock
|
||||
# attribute must not become the pool size.
|
||||
store, _, _, _ = _make_store()
|
||||
assert store._pool_max_size == 8
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestWritePathEnsuresSchemaOnce:
|
||||
def test_add_texts_ensures_once_across_two_calls(self):
|
||||
store, _, mock_cursor, _ = _make_store()
|
||||
mock_cursor.fetchone.return_value = (1,)
|
||||
|
||||
store.add_texts(["a"])
|
||||
store.add_texts(["b"])
|
||||
|
||||
assert store._ensure_table_exists.call_count == 1
|
||||
|
||||
def test_add_chunk_ensures_once_across_two_calls(self):
|
||||
store, _, mock_cursor, _ = _make_store()
|
||||
mock_cursor.fetchone.return_value = (1,)
|
||||
|
||||
store.add_chunk("a")
|
||||
store.add_chunk("b")
|
||||
|
||||
assert store._ensure_table_exists.call_count == 1
|
||||
|
||||
def test_reads_and_deletes_never_ensure_schema(self):
|
||||
store, _, mock_cursor, _ = _make_store()
|
||||
mock_cursor.fetchall.return_value = []
|
||||
mock_cursor.fetchone.return_value = (0,)
|
||||
mock_cursor.rowcount = 0
|
||||
|
||||
store.search("q")
|
||||
store.keyword_search("q")
|
||||
store.get_chunks()
|
||||
store.delete_index()
|
||||
store.delete_chunk("1")
|
||||
|
||||
store._ensure_table_exists.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestCreateSchema:
|
||||
def test_emits_the_ddl_and_leaves_the_commit_to_the_caller(self):
|
||||
from application.vectorstore.pgvector import PGVectorStore
|
||||
|
||||
conn, cursor = MagicMock(), MagicMock()
|
||||
conn.cursor.return_value = cursor
|
||||
|
||||
PGVectorStore.create_schema(conn, dimension=8)
|
||||
|
||||
statements = " ".join(str(c) for c in cursor.execute.call_args_list)
|
||||
assert "CREATE EXTENSION IF NOT EXISTS vector" in statements
|
||||
assert "CREATE TABLE IF NOT EXISTS documents" in statements
|
||||
assert "vector(8)" in statements
|
||||
assert "documents_source_id_idx" in statements
|
||||
assert "documents_text_fts_idx" in statements
|
||||
assert cursor.execute.call_count == 4
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
def test_ensure_table_exists_locks_then_commits(self):
|
||||
store, mock_conn, mock_cursor, _ = _make_store()
|
||||
del store._ensure_table_exists # exercise the real method
|
||||
|
||||
store._ensure_table_exists()
|
||||
|
||||
statements = " ".join(str(c) for c in mock_cursor.execute.call_args_list)
|
||||
assert "pg_advisory_xact_lock" in statements
|
||||
assert "CREATE TABLE IF NOT EXISTS documents" in statements
|
||||
mock_conn.commit.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestTableDimension:
|
||||
def _conn(self, rows):
|
||||
conn, cursor = MagicMock(), MagicMock()
|
||||
conn.cursor.return_value = cursor
|
||||
cursor.fetchone.side_effect = rows
|
||||
return conn
|
||||
|
||||
def test_parses_the_declared_vector_width(self):
|
||||
from application.vectorstore.pgvector import PGVectorStore
|
||||
|
||||
conn = self._conn([("documents",), ("vector(768)",)])
|
||||
assert PGVectorStore.table_dimension(conn) == 768
|
||||
|
||||
def test_returns_none_when_the_table_is_absent(self):
|
||||
from application.vectorstore.pgvector import PGVectorStore
|
||||
|
||||
conn = self._conn([(None,)])
|
||||
assert PGVectorStore.table_dimension(conn) is None
|
||||
|
||||
def test_returns_none_when_the_column_type_is_not_a_vector(self):
|
||||
from application.vectorstore.pgvector import PGVectorStore
|
||||
|
||||
conn = self._conn([("documents",), ("text",)])
|
||||
assert PGVectorStore.table_dimension(conn) is None
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestReadPathsCloseTheirTransaction:
|
||||
"""A read that never commits leaves the connection idle-in-transaction.
|
||||
|
||||
On a persistent (now pooled) connection that holds a snapshot open for the
|
||||
life of the process and makes psycopg_pool roll back every returned
|
||||
connection.
|
||||
"""
|
||||
|
||||
def test_search_commits(self):
|
||||
store, mock_conn, mock_cursor, _ = _make_store()
|
||||
mock_cursor.fetchall.return_value = [("hello", {}, 0.1), ("bye", {}, 0.2)]
|
||||
|
||||
store.search_with_scores("q", k=2)
|
||||
|
||||
mock_conn.commit.assert_called_once()
|
||||
|
||||
def test_keyword_search_commits(self):
|
||||
store, mock_conn, mock_cursor, _ = _make_store()
|
||||
mock_cursor.fetchall.return_value = []
|
||||
|
||||
store.keyword_search("q")
|
||||
|
||||
mock_conn.commit.assert_called_once()
|
||||
|
||||
def test_get_chunks_commits(self):
|
||||
store, mock_conn, mock_cursor, _ = _make_store()
|
||||
mock_cursor.fetchall.return_value = []
|
||||
|
||||
store.get_chunks()
|
||||
|
||||
mock_conn.commit.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestConnectionPooling:
|
||||
def _pooled_store(self, monkeypatch):
|
||||
store, _, _, _ = _make_store()
|
||||
store._connection = None
|
||||
store._pool_max_size = 4
|
||||
pooled_conn = MagicMock()
|
||||
pooled_conn.closed = False
|
||||
fake_pool = MagicMock()
|
||||
fake_pool.getconn.return_value = pooled_conn
|
||||
monkeypatch.setattr(
|
||||
pgvector_module, "_pool_for", lambda dsn, size: fake_pool
|
||||
)
|
||||
return store, fake_pool, pooled_conn
|
||||
|
||||
def test_get_connection_checks_out_of_the_pool(self, monkeypatch):
|
||||
store, fake_pool, pooled_conn = self._pooled_store(monkeypatch)
|
||||
|
||||
conn = store._get_connection()
|
||||
|
||||
assert conn is pooled_conn
|
||||
assert store._pooled is True
|
||||
fake_pool.getconn.assert_called_once()
|
||||
store._psycopg.connect.assert_not_called()
|
||||
|
||||
def test_probes_are_applied_once_per_physical_connection(self, monkeypatch):
|
||||
store, _, pooled_conn = self._pooled_store(monkeypatch)
|
||||
applied = Mock()
|
||||
monkeypatch.setattr(store, "_apply_ivfflat_probes", applied)
|
||||
|
||||
store._get_connection()
|
||||
store._connection = None # simulate a second checkout of the same conn
|
||||
store._get_connection()
|
||||
|
||||
assert applied.call_count == 1
|
||||
|
||||
def test_close_rolls_back_and_returns_the_connection(self, monkeypatch):
|
||||
store, fake_pool, pooled_conn = self._pooled_store(monkeypatch)
|
||||
store._get_connection()
|
||||
pooled_conn.info.transaction_status.name = "INTRANS"
|
||||
monkeypatch.setitem(
|
||||
pgvector_module._POOLS, store._connection_string, fake_pool
|
||||
)
|
||||
|
||||
store.close()
|
||||
|
||||
pooled_conn.rollback.assert_called_once()
|
||||
fake_pool.putconn.assert_called_once_with(pooled_conn)
|
||||
pooled_conn.close.assert_not_called()
|
||||
assert store._connection is None
|
||||
|
||||
def test_close_does_not_roll_back_an_idle_connection(self, monkeypatch):
|
||||
store, fake_pool, pooled_conn = self._pooled_store(monkeypatch)
|
||||
store._get_connection()
|
||||
pooled_conn.info.transaction_status.name = "IDLE"
|
||||
monkeypatch.setitem(
|
||||
pgvector_module._POOLS, store._connection_string, fake_pool
|
||||
)
|
||||
|
||||
store.close()
|
||||
|
||||
pooled_conn.rollback.assert_not_called()
|
||||
fake_pool.putconn.assert_called_once_with(pooled_conn)
|
||||
|
||||
def test_legacy_path_connects_directly_and_closes(self):
|
||||
store, mock_conn, _, _ = _make_store()
|
||||
store._pool_max_size = 0
|
||||
mock_conn.closed = True
|
||||
new_conn = MagicMock()
|
||||
new_conn.closed = False
|
||||
store._psycopg.connect.return_value = new_conn
|
||||
|
||||
conn = store._get_connection()
|
||||
assert conn is new_conn
|
||||
assert store._pooled is False
|
||||
|
||||
store.close()
|
||||
new_conn.close.assert_called_once()
|
||||
|
||||
def test_del_never_raises(self):
|
||||
store, mock_conn, _, _ = _make_store()
|
||||
mock_conn.close.side_effect = RuntimeError("already gone")
|
||||
|
||||
store.__del__() # must not propagate
|
||||
Reference in new issue
Block a user