tiennm99/DocsGPT
1
0 Fork 0
mirror of https://github.com/tiennm99/DocsGPT.git synced 2026-10-18 14:13:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: rename the application package to docsgpt

Browse Source 
The backend import package is now docsgpt, the name it will carry on PyPI;
application was far too generic to install into anyone's site-packages.
git mv plus a mechanical rewrite of every import, dotted string and path
reference: 734 Python files, the compose files, Dockerfile, workflows, docs,
setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage
config, .gitignore. Behaviour is unchanged.

Kept for one release:
- A top-level application package whose meta-path finder resolves
  application.x.y to the already-imported docsgpt.x.y object, so old imports
  and entry points (celery -A application.app.celery,
  uvicorn application.asgi:asgi_app) keep working with a FutureWarning.
- Celery registers every application.* task name as an alias of its
  docsgpt.* task on start-up, so messages queued by the previous release still
  run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries
  the previous release wrote are left unread instead of firing twice.

The backend image builds from the repository root (docker build -f
docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore
allow-lists docsgpt/ and application/ and keeps caches, local data, .env
files, the sample index files and the Dockerfile out. Compose and the image
workflows point at the new context.
This commit is contained in:
Alex committed 2026-09-07 10:20:43 +01:00
1 parent 3e7f1f91b4
commit 574f96341e
984 files changed
+9847 -9682
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/worker/conftest.py
+4 -4
View File
@@ -1,6 +1,6 @@
"""Fixtures for Celery worker smoke tests.
These tests exercise the task *bodies* in ``application.worker`` against a
These tests exercise the task *bodies* in ``docsgpt.worker`` against a
real Postgres schema (via the ephemeral ``pg_conn`` fixture from the root
``tests/conftest.py``). External I/O — storage, the embedding pipeline,
the retriever, the LLM, the backend HTTP callback — is mocked, but every
@@ -20,7 +20,7 @@ from sqlalchemy import Connection
@pytest.fixture
def patch_worker_db(pg_conn, monkeypatch):
"""Redirect ``db_session`` / ``db_readonly`` in ``application.worker``.
"""Redirect ``db_session`` / ``db_readonly`` in ``docsgpt.worker``.
Both helpers yield the per-test transactional ``pg_conn``, so any
writes a task performs are visible to the test and roll back on
@@ -32,8 +32,8 @@ def patch_worker_db(pg_conn, monkeypatch):
def _use_pg_conn() -> Iterator[Connection]:
yield pg_conn
monkeypatch.setattr("application.worker.db_session", _use_pg_conn)
monkeypatch.setattr("application.worker.db_readonly", _use_pg_conn)
monkeypatch.setattr("docsgpt.worker.db_session", _use_pg_conn)
monkeypatch.setattr("docsgpt.worker.db_readonly", _use_pg_conn)
@pytest.fixture
tests/worker/test_agent_workers.py
+32 -32
View File
@@ -16,7 +16,7 @@ from unittest.mock import MagicMock
import pytest
from application.storage.db.repositories.agents import AgentsRepository
from docsgpt.storage.db.repositories.agents import AgentsRepository
@pytest.mark.unit
@@ -24,8 +24,8 @@ class TestAgentWebhookWorker:
def test_resolves_agent_by_uuid_and_runs_logic(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from application.agents import headless_runner
from docsgpt import worker
from docsgpt.agents import headless_runner
agent = AgentsRepository(pg_conn).create(
user_id="alice",
@@ -79,8 +79,8 @@ class TestAgentWebhookWorker:
def test_missing_agent_raises(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from application.agents import headless_runner
from docsgpt import worker
from docsgpt.agents import headless_runner
monkeypatch.setattr(
headless_runner, "run_agent_headless", lambda *a, **k: {},
@@ -92,9 +92,9 @@ class TestAgentWebhookWorker:
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
"""Headless runner errors must raise — a returned dict reads as success."""
from application import worker
from application.agents import headless_runner
from application.storage.db.repositories.agents import AgentsRepository
from docsgpt import worker
from docsgpt.agents import headless_runner
from docsgpt.storage.db.repositories.agents import AgentsRepository
agent = AgentsRepository(pg_conn).create(
user_id="alice", name="hook-agent", status="active",
@@ -117,14 +117,14 @@ class TestAgentWebhookWorker:
from contextlib import contextmanager
from types import SimpleNamespace
from application import worker
from application.agents import headless_runner
from application.agents.tool_executor import ToolExecutor
from application.llm.handlers.base import (
from docsgpt import worker
from docsgpt.agents import headless_runner
from docsgpt.agents.tool_executor import ToolExecutor
from docsgpt.llm.handlers.base import (
LLMHandler,
ToolCall,
)
from application.storage.db.repositories.agents import AgentsRepository
from docsgpt.storage.db.repositories.agents import AgentsRepository
agent = AgentsRepository(pg_conn).create(
user_id="alice", name="hook-agent", status="active",
@@ -138,36 +138,36 @@ class TestAgentWebhookWorker:
yield pg_conn
monkeypatch.setattr(
"application.agents.tool_executor.db_session", _use_pg_conn,
"docsgpt.agents.tool_executor.db_session", _use_pg_conn,
)
# Stub model resolution + retriever so the call threads through.
monkeypatch.setattr(
"application.core.model_utils.get_default_model_id",
"docsgpt.core.model_utils.get_default_model_id",
lambda: "gpt-4",
)
monkeypatch.setattr(
"application.core.model_utils.validate_model_id",
"docsgpt.core.model_utils.validate_model_id",
lambda m, **_kwargs: True,
)
monkeypatch.setattr(
"application.core.model_utils.get_provider_from_model_id",
"docsgpt.core.model_utils.get_provider_from_model_id",
lambda m, **_kwargs: "openai",
)
monkeypatch.setattr(
"application.core.model_utils.get_api_key_for_provider",
"docsgpt.core.model_utils.get_api_key_for_provider",
lambda p: "sk-test",
)
monkeypatch.setattr(
"application.utils.calculate_doc_token_budget",
"docsgpt.utils.calculate_doc_token_budget",
lambda model_id=None, **_kwargs: 1000,
)
monkeypatch.setattr(
"application.api.answer.services.stream_processor.get_prompt",
"docsgpt.api.answer.services.stream_processor.get_prompt",
lambda prompt_id: "prompt text",
)
monkeypatch.setattr(
"application.retriever.retriever_creator.RetrieverCreator.create_retriever",
"docsgpt.retriever.retriever_creator.RetrieverCreator.create_retriever",
lambda *a, **kw: SimpleNamespace(search=lambda q: []),
)
@@ -235,7 +235,7 @@ class TestAgentWebhookWorker:
return _FakeAgent()
monkeypatch.setattr(
"application.agents.agent_creator.AgentCreator.create_agent",
"docsgpt.agents.agent_creator.AgentCreator.create_agent",
_fake_agent_factory,
)
monkeypatch.setattr(headless_runner, "db_readonly", _use_pg_conn)
@@ -268,8 +268,8 @@ class TestRunAgentHeadlessFromWebhook:
"""Smoke-test that run_agent_headless reads the source row from PG."""
from contextlib import contextmanager
from application.agents import headless_runner
from application.storage.db.repositories.sources import SourcesRepository
from docsgpt.agents import headless_runner
from docsgpt.storage.db.repositories.sources import SourcesRepository
@contextmanager
def _use_pg_conn():
@@ -286,25 +286,25 @@ class TestRunAgentHeadlessFromWebhook:
# Silence model/provider resolution so we don't need a real key.
monkeypatch.setattr(
"application.core.model_utils.get_default_model_id", lambda: "gpt-4"
"docsgpt.core.model_utils.get_default_model_id", lambda: "gpt-4"
)
monkeypatch.setattr(
"application.core.model_utils.validate_model_id", lambda m, **_kwargs: True
"docsgpt.core.model_utils.validate_model_id", lambda m, **_kwargs: True
)
monkeypatch.setattr(
"application.core.model_utils.get_provider_from_model_id",
"docsgpt.core.model_utils.get_provider_from_model_id",
lambda m, **_kwargs: "openai",
)
monkeypatch.setattr(
"application.core.model_utils.get_api_key_for_provider",
"docsgpt.core.model_utils.get_api_key_for_provider",
lambda p: "sk-test",
)
monkeypatch.setattr(
"application.utils.calculate_doc_token_budget",
"docsgpt.utils.calculate_doc_token_budget",
lambda model_id=None, **_kwargs: 1000,
)
monkeypatch.setattr(
"application.api.answer.services.stream_processor.get_prompt",
"docsgpt.api.answer.services.stream_processor.get_prompt",
lambda prompt_id: "prompt text",
)
@@ -320,7 +320,7 @@ class TestRunAgentHeadlessFromWebhook:
return []
monkeypatch.setattr(
"application.retriever.retriever_creator.RetrieverCreator.create_retriever",
"docsgpt.retriever.retriever_creator.RetrieverCreator.create_retriever",
lambda *a, **kw: _FakeRetriever(**kw),
)
@@ -328,7 +328,7 @@ class TestRunAgentHeadlessFromWebhook:
fake_agent.gen.return_value = iter([{"answer": "done"}])
fake_agent.current_token_count = 0
monkeypatch.setattr(
"application.agents.agent_creator.AgentCreator.create_agent",
"docsgpt.agents.agent_creator.AgentCreator.create_agent",
lambda *a, **kw: fake_agent,
)
tests/worker/test_attachment_worker.py
+33 -33
View File
@@ -1,4 +1,4 @@
"""Smoke test for ``application.worker.attachment_worker``.
"""Smoke test for ``docsgpt.worker.attachment_worker``.
The happy path parses an uploaded file and inserts a row into
``attachments``. We mock the parser boundary (``StorageCreator.get_storage``
@@ -14,8 +14,8 @@ from unittest.mock import MagicMock
import pytest
from application.parser.schema.base import Document
from application.storage.db.repositories.attachments import AttachmentsRepository
from docsgpt.parser.schema.base import Document
from docsgpt.storage.db.repositories.attachments import AttachmentsRepository
@pytest.mark.unit
@@ -23,7 +23,7 @@ class TestAttachmentWorker:
def test_inserts_row_in_attachments(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
fake_doc = Document(
text="hello world",
@@ -80,8 +80,8 @@ class TestAttachmentWorker:
records the error; ``content`` stays NULL so the model can never
read a traceback as the document.
"""
from application import worker
from application.parser.file.base_parser import DocumentParseError
from docsgpt import worker
from docsgpt.parser.file.base_parser import DocumentParseError
published: list[tuple[str, dict]] = []
monkeypatch.setattr(
@@ -137,8 +137,8 @@ class TestAttachmentWorker:
returning its traceback as content, an unguarded task would turn one
unreadable upload into a retry loop of identical failures.
"""
from application.api.user import tasks as user_tasks
from application.parser.file.base_parser import DocumentParseError
from docsgpt.api.user import tasks as user_tasks
from docsgpt.parser.file.base_parser import DocumentParseError
task = getattr(user_tasks, task_name)
assert DocumentParseError in task.dont_autoretry_for
@@ -161,8 +161,8 @@ class TestBoundedAttachmentCopy:
return path
def test_oversized_csv_is_copied_and_truncated(self, tmp_path, monkeypatch):
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "ATTACHMENT_TEXT_MAX_BYTES", 1024)
original = self._write(
@@ -183,8 +183,8 @@ class TestBoundedAttachmentCopy:
Path(parse_path).unlink()
def test_small_file_returned_as_is(self, tmp_path, monkeypatch):
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "ATTACHMENT_TEXT_MAX_BYTES", 1024)
original = self._write(tmp_path, "small.csv", b"a,b\n1,2\n")
@@ -195,8 +195,8 @@ class TestBoundedAttachmentCopy:
assert is_temp is False
def test_non_text_suffix_is_never_truncated(self, tmp_path, monkeypatch):
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "ATTACHMENT_TEXT_MAX_BYTES", 64)
original = self._write(tmp_path, "doc.pdf", b"%PDF-1.7 " + b"x" * 500)
@@ -207,8 +207,8 @@ class TestBoundedAttachmentCopy:
assert is_temp is False
def test_cap_zero_disables_truncation(self, tmp_path, monkeypatch):
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "ATTACHMENT_TEXT_MAX_BYTES", 0)
original = self._write(tmp_path, "big.csv", b"1,2\n" * 1000)
@@ -221,8 +221,8 @@ class TestBoundedAttachmentCopy:
def test_single_line_without_newline_falls_back_to_hard_cut(
self, tmp_path, monkeypatch
):
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "ATTACHMENT_TEXT_MAX_BYTES", 256)
original = self._write(tmp_path, "oneline.txt", b"x" * 5000)
@@ -241,8 +241,8 @@ class TestBoundedAttachmentCopy:
write a one-byte copy and throw the attachment away — the partial
final line is the better trade.
"""
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
monkeypatch.setattr(settings, "ATTACHMENT_TEXT_MAX_BYTES", 256)
original = self._write(tmp_path, "leading.log", b"\n" + b"x" * 5000)
@@ -272,8 +272,8 @@ class TestAttachmentZipBombGuard:
wb.save(str(path))
def test_rejects_when_inner_size_exceeds_cap(self, tmp_path, monkeypatch):
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
path = tmp_path / "book.xlsx"
self._make_xlsx(path)
@@ -283,8 +283,8 @@ class TestAttachmentZipBombGuard:
worker._reject_attachment_zip_bomb(str(path))
def test_rejects_when_too_many_entries(self, tmp_path, monkeypatch):
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
path = tmp_path / "book.xlsx"
self._make_xlsx(path)
@@ -294,8 +294,8 @@ class TestAttachmentZipBombGuard:
worker._reject_attachment_zip_bomb(str(path))
def test_allows_reasonable_archive(self, tmp_path, monkeypatch):
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
path = tmp_path / "book.xlsx"
self._make_xlsx(path)
@@ -306,8 +306,8 @@ class TestAttachmentZipBombGuard:
worker._reject_attachment_zip_bomb(str(path))
def test_non_container_suffix_is_ignored(self, tmp_path, monkeypatch):
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
path = tmp_path / "notes.txt"
path.write_bytes(b"x" * 5000)
@@ -317,8 +317,8 @@ class TestAttachmentZipBombGuard:
worker._reject_attachment_zip_bomb(str(path))
def test_corrupt_zip_is_left_to_the_parser(self, tmp_path, monkeypatch):
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
path = tmp_path / "broken.xlsx"
path.write_bytes(b"not a real zip")
@@ -342,7 +342,7 @@ class TestAttachmentTypeGuard:
def test_binary_without_a_parser_fails_instead_of_being_read_as_text(
self, pg_conn, patch_worker_db, task_self, monkeypatch, tmp_path
):
from application import worker
from docsgpt import worker
local_path = tmp_path / "clip.mp4"
local_path.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomavc1")
@@ -388,7 +388,7 @@ class TestAttachmentTypeGuard:
.webp the route admitted on its name would otherwise be opened as
plain text here.
"""
from application import worker
from docsgpt import worker
local_path = tmp_path / "scan.webp"
local_path.write_bytes(b"RIFF\x00\x00\x00\x00WEBPVP8 " + bytes(range(256)))
@@ -429,7 +429,7 @@ class TestAttachmentTypeGuard:
def test_text_without_a_parser_is_parsed(
self, pg_conn, patch_worker_db, task_self, monkeypatch, tmp_path
):
from application import worker
from docsgpt import worker
local_path = tmp_path / "server.log"
local_path.write_text("2026-09-02 ERROR boom\n", encoding="utf-8")
tests/worker/test_attachment_worker_provenance.py
+17 -17
View File
@@ -14,11 +14,11 @@ from unittest.mock import patch
import pytest
import application.storage.db.engine as engine_module
from application.parser.file.base_parser import DocumentParseError
from application.storage.db.repositories.attachments import AttachmentsRepository
from application.storage.db.session import db_readonly
from application.utils import get_encoding
import docsgpt.storage.db.engine as engine_module
from docsgpt.parser.file.base_parser import DocumentParseError
from docsgpt.storage.db.repositories.attachments import AttachmentsRepository
from docsgpt.storage.db.session import db_readonly
from docsgpt.utils import get_encoding
class _StubTask:
@@ -45,18 +45,18 @@ def wired_engine(pg_engine, monkeypatch):
@pytest.fixture()
def storage_dir(tmp_path, monkeypatch):
"""LocalStorage rooted at tmp_path, patched into the worker."""
from application.storage.local import LocalStorage
from docsgpt.storage.local import LocalStorage
storage = LocalStorage(base_dir=str(tmp_path))
monkeypatch.setattr(
"application.storage.storage_creator.StorageCreator.get_storage",
"docsgpt.storage.storage_creator.StorageCreator.get_storage",
classmethod(lambda cls: storage),
)
return tmp_path
def _run_worker(file_info, user="prov-user"):
from application.worker import attachment_worker
from docsgpt.worker import attachment_worker
return attachment_worker(_StubTask(), file_info, user)
@@ -123,7 +123,7 @@ class TestTruncationProvenance:
dense = "統計資料表格內容分析、報告書類文書處理系統。設計開發運用管理。\n" * 10000
info = _file_info(storage_dir)
monkeypatch.setattr(
"application.worker.SimpleDirectoryReader",
"docsgpt.worker.SimpleDirectoryReader",
lambda **kwargs: type("R", (), {"load_data": lambda self: [_Doc(dense)]})(),
)
@@ -145,7 +145,7 @@ class TestTruncationProvenance:
text = "plain short attachment content"
info = _file_info(storage_dir)
monkeypatch.setattr(
"application.worker.SimpleDirectoryReader",
"docsgpt.worker.SimpleDirectoryReader",
lambda **kwargs: type("R", (), {"load_data": lambda self: [_Doc(text)]})(),
)
@@ -165,7 +165,7 @@ class TestFailureProvenance:
def _raise(**kwargs):
raise DocumentParseError("Failed to parse broken.xlsx with docling: boom")
monkeypatch.setattr("application.worker.SimpleDirectoryReader", _raise)
monkeypatch.setattr("docsgpt.worker.SimpleDirectoryReader", _raise)
with pytest.raises(DocumentParseError):
_run_worker(info)
@@ -190,7 +190,7 @@ class TestFailureProvenance:
raise RuntimeError("transient blip")
return type("R", (), {"load_data": lambda self: [_Doc("recovered fine")]})()
monkeypatch.setattr("application.worker.SimpleDirectoryReader", _flaky)
monkeypatch.setattr("docsgpt.worker.SimpleDirectoryReader", _flaky)
with pytest.raises(RuntimeError):
_run_worker(info)
@@ -208,7 +208,7 @@ class TestFailureProvenance:
# An exception after the success write (e.g. event publishing) must
# not replace stored content with a NULL-content failed row; the
# extraction result is already durable.
from application.worker import record_attachment_failure
from docsgpt.worker import record_attachment_failure
info = _file_info(storage_dir)
_run_worker(info)
@@ -227,9 +227,9 @@ class TestFailureProvenance:
def _raise(**kwargs):
raise DocumentParseError("original parse error")
monkeypatch.setattr("application.worker.SimpleDirectoryReader", _raise)
monkeypatch.setattr("docsgpt.worker.SimpleDirectoryReader", _raise)
monkeypatch.setattr(
"application.worker.db_session",
"docsgpt.worker.db_session",
_raising_db_session,
)
@@ -244,7 +244,7 @@ def _raising_db_session():
@pytest.mark.usefixtures("wired_engine")
class TestPoisonProvenance:
def test_poison_guard_writes_failed_row(self):
from application.api.user.tasks import _emit_attachment_poison_event
from docsgpt.api.user.tasks import _emit_attachment_poison_event
attachment_id = str(uuid.uuid4())
bound = {
@@ -256,7 +256,7 @@ class TestPoisonProvenance:
},
}
with patch("application.events.publisher.publish_user_event"):
with patch("docsgpt.events.publisher.publish_user_event"):
_emit_attachment_poison_event("store_attachment", bound)
row = _fetch(attachment_id)
tests/worker/test_convert_source_to_wiki.py
+26 -26
View File
@@ -1,4 +1,4 @@
"""Tests for ``application.worker.convert_source_to_wiki_worker``.
"""Tests for ``docsgpt.worker.convert_source_to_wiki_worker``.
The worker reassembles wiki pages from a source's existing vector-store
chunks (grouped by ``metadata.source``) and enqueues a per-page re-embed.
@@ -13,8 +13,8 @@ from unittest.mock import MagicMock
import pytest
from application.storage.db.repositories.sources import SourcesRepository
from application.storage.db.repositories.wiki_pages import WikiPagesRepository
from docsgpt.storage.db.repositories.sources import SourcesRepository
from docsgpt.storage.db.repositories.wiki_pages import WikiPagesRepository
def _seed_source(pg_conn, config=None, file_path=""):
@@ -41,7 +41,7 @@ def _patch_store(monkeypatch, chunks):
store = MagicMock(name="vectorstore")
store.get_chunks.return_value = chunks
monkeypatch.setattr(
"application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
"docsgpt.vectorstore.vector_creator.VectorCreator.create_vectorstore",
lambda *a, **kw: store,
)
return store
@@ -49,7 +49,7 @@ def _patch_store(monkeypatch, chunks):
def _patch_reembed(monkeypatch):
delay = MagicMock(name="reembed_delay")
monkeypatch.setattr("application.api.user.tasks.reembed_wiki_page.delay", delay)
monkeypatch.setattr("docsgpt.api.user.tasks.reembed_wiki_page.delay", delay)
return delay
@@ -58,7 +58,7 @@ class TestConvertSourceToWikiWorker:
def test_two_pages_reassembled_from_chunks(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(
@@ -97,7 +97,7 @@ class TestConvertSourceToWikiWorker:
def test_original_chunks_deleted_after_convert(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
store = _patch_store(
@@ -120,7 +120,7 @@ class TestConvertSourceToWikiWorker:
def test_chunk_without_doc_id_not_deleted(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
chunk = {"text": "body", "metadata": {"source": "a.md"}}
@@ -137,7 +137,7 @@ class TestConvertSourceToWikiWorker:
):
# A chunk skipped for an invalid path must still have its original
# vector chunk purged, not left orphaned after the source flips to wiki.
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
store = _patch_store(
@@ -158,7 +158,7 @@ class TestConvertSourceToWikiWorker:
def test_no_pages_path_deletes_nothing(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
store = _patch_store(monkeypatch, [])
@@ -172,8 +172,8 @@ class TestConvertSourceToWikiWorker:
def test_kind_flipped_and_exposure_defaulted(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from application.storage.db.source_config import SourceConfig
from docsgpt import worker
from docsgpt.storage.db.source_config import SourceConfig
source_id = _seed_source(pg_conn)
_patch_store(monkeypatch, [_chunk("body", source="a.md")])
@@ -189,8 +189,8 @@ class TestConvertSourceToWikiWorker:
def test_preserves_non_default_exposure(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from application.storage.db.source_config import SourceConfig
from docsgpt import worker
from docsgpt.storage.db.source_config import SourceConfig
source_id = _seed_source(
pg_conn, config={"retrieval": {"exposure": "agentic_tool"}}
@@ -208,7 +208,7 @@ class TestConvertSourceToWikiWorker:
def test_chunk_order_hint_respected(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(
@@ -229,7 +229,7 @@ class TestConvertSourceToWikiWorker:
def test_short_overlap_not_trimmed(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(
@@ -250,7 +250,7 @@ class TestConvertSourceToWikiWorker:
def test_long_overlap_trimmed_between_chunks(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
overlap = "x" * 40
source_id = _seed_source(pg_conn)
@@ -272,8 +272,8 @@ class TestConvertSourceToWikiWorker:
def test_no_chunks_leaves_kind_and_structure(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from application.storage.db.source_config import SourceConfig
from docsgpt import worker
from docsgpt.storage.db.source_config import SourceConfig
original_structure = {"a.md": {"type": "text/markdown", "size_bytes": 1}}
source_id = _seed_source(pg_conn)
@@ -296,7 +296,7 @@ class TestConvertSourceToWikiWorker:
def test_missing_path_chunk_skipped(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(
@@ -318,7 +318,7 @@ class TestConvertSourceToWikiWorker:
def test_invalid_path_chunk_skipped(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(
@@ -341,7 +341,7 @@ class TestConvertSourceToWikiWorker:
def test_path_falls_back_to_filename_then_title(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(
@@ -362,7 +362,7 @@ class TestConvertSourceToWikiWorker:
def test_crawler_chunk_uses_file_path_not_url(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(
@@ -387,7 +387,7 @@ class TestConvertSourceToWikiWorker:
def test_connector_chunks_kept_separate_by_file_name(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(
@@ -408,7 +408,7 @@ class TestConvertSourceToWikiWorker:
def test_url_only_chunk_normalized_not_skipped(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(
@@ -427,7 +427,7 @@ class TestConvertSourceToWikiWorker:
def test_already_wiki_returns_early(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn, config={"kind": "wiki"})
store = _patch_store(monkeypatch, [_chunk("body", source="a.md")])
tests/worker/test_extract_graph.py
+18 -18
View File
@@ -1,4 +1,4 @@
"""Tests for ``application.worker.extract_graph_worker``.
"""Tests for ``docsgpt.worker.extract_graph_worker``.
The worker loads the source row, fetches its chunks from the vector store, and
delegates to ``extract_graph_for_source``. ``graphrag_available``,
@@ -13,7 +13,7 @@ from unittest.mock import MagicMock
import pytest
from application.storage.db.repositories.sources import SourcesRepository
from docsgpt.storage.db.repositories.sources import SourcesRepository
def _seed_source(pg_conn, config=None):
@@ -31,7 +31,7 @@ def _patch_store(monkeypatch, chunks):
store = MagicMock(name="vectorstore")
store.get_chunks.return_value = chunks
monkeypatch.setattr(
"application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
"docsgpt.vectorstore.vector_creator.VectorCreator.create_vectorstore",
lambda *a, **kw: store,
)
return store
@@ -42,7 +42,7 @@ class TestExtractGraphWorker:
def test_fetches_chunks_and_calls_extraction(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
chunks = [
@@ -52,14 +52,14 @@ class TestExtractGraphWorker:
store = _patch_store(monkeypatch, chunks)
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: True
"docsgpt.graphrag.graphrag_available", lambda: True
)
extract = MagicMock(
name="extract_graph_for_source",
return_value={"nodes": 3, "edges": 2, "chunks_processed": 2},
)
monkeypatch.setattr(
"application.graphrag.extraction.extract_graph_for_source", extract
"docsgpt.graphrag.extraction.extract_graph_for_source", extract
)
result = worker.extract_graph_worker(task_self, source_id, "alice")
@@ -75,15 +75,15 @@ class TestExtractGraphWorker:
def test_unavailable_returns_status_no_extraction(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
store = _patch_store(monkeypatch, [])
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: False
"docsgpt.graphrag.graphrag_available", lambda: False
)
extract = MagicMock(name="extract_graph_for_source")
monkeypatch.setattr(
"application.graphrag.extraction.extract_graph_for_source", extract
"docsgpt.graphrag.extraction.extract_graph_for_source", extract
)
result = worker.extract_graph_worker(task_self, "src-x", "alice")
@@ -95,19 +95,19 @@ class TestExtractGraphWorker:
def test_empty_chunks_still_calls_extraction(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(monkeypatch, [])
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: True
"docsgpt.graphrag.graphrag_available", lambda: True
)
extract = MagicMock(
name="extract_graph_for_source",
return_value={"nodes": 0, "edges": 0, "chunks_processed": 0},
)
monkeypatch.setattr(
"application.graphrag.extraction.extract_graph_for_source", extract
"docsgpt.graphrag.extraction.extract_graph_for_source", extract
)
result = worker.extract_graph_worker(task_self, source_id, "alice")
@@ -119,15 +119,15 @@ class TestExtractGraphWorker:
def test_publishes_completed_event(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(monkeypatch, [{"doc_id": "c1", "text": "alpha"}])
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: True
"docsgpt.graphrag.graphrag_available", lambda: True
)
monkeypatch.setattr(
"application.graphrag.extraction.extract_graph_for_source",
"docsgpt.graphrag.extraction.extract_graph_for_source",
MagicMock(return_value={"nodes": 1, "edges": 0, "chunks_processed": 1}),
)
events = []
@@ -146,19 +146,19 @@ class TestExtractGraphWorker:
def test_publishes_failed_event_on_error(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(monkeypatch, [{"doc_id": "c1", "text": "alpha"}])
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: True
"docsgpt.graphrag.graphrag_available", lambda: True
)
def _boom(*a, **kw):
raise RuntimeError("extraction blew up")
monkeypatch.setattr(
"application.graphrag.extraction.extract_graph_for_source", _boom
"docsgpt.graphrag.extraction.extract_graph_for_source", _boom
)
events = []
monkeypatch.setattr(
tests/worker/test_graph_extraction_enqueue.py
+23 -23
View File
@@ -14,13 +14,13 @@ from unittest.mock import MagicMock
import pytest
from application.parser.schema.base import Document
from application.storage.db.repositories.sources import SourcesRepository
from docsgpt.parser.schema.base import Document
from docsgpt.storage.db.repositories.sources import SourcesRepository
@pytest.fixture
def _mock_remote_pipeline(monkeypatch):
from application import worker
from docsgpt import worker
fake_loader = MagicMock(name="remote_loader")
fake_loader.load_data.return_value = [
@@ -53,7 +53,7 @@ def _mock_remote_pipeline(monkeypatch):
def _patch_delay(monkeypatch):
delay = MagicMock(name="extract_graph_delay")
monkeypatch.setattr(
"application.api.user.tasks.extract_graph.delay", delay
"docsgpt.api.user.tasks.extract_graph.delay", delay
)
return delay
@@ -68,7 +68,7 @@ def _seed_source(pg_conn, user, config):
@pytest.mark.unit
class TestGraphExtractionKey:
def test_shape_and_state_sensitivity(self):
from application.worker import _source_updated_at, graph_extraction_key
from docsgpt.worker import _source_updated_at, graph_extraction_key
sid = "11111111-1111-1111-1111-111111111111"
key_a = graph_extraction_key(
@@ -88,7 +88,7 @@ class TestGraphExtractionKey:
assert key_a == key_a_again
def test_falls_back_to_date(self):
from application.worker import _source_updated_at
from docsgpt.worker import _source_updated_at
assert _source_updated_at({"date": "2026-06-23T00:00:00+00:00"}) == (
"2026-06-23T00:00:00+00:00"
@@ -103,10 +103,10 @@ class TestRemoteWorkerEnqueuesGraphExtraction:
self, task_self, pg_conn, patch_worker_db, monkeypatch,
_mock_remote_pipeline,
):
from application import worker
from docsgpt import worker
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: True
"docsgpt.graphrag.graphrag_available", lambda: True
)
delay = _patch_delay(monkeypatch)
@@ -137,10 +137,10 @@ class TestRemoteWorkerEnqueuesGraphExtraction:
_mock_remote_pipeline,
):
"""Two enqueues for the same source state share a key (concurrent dups)."""
from application import worker
from docsgpt import worker
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: True
"docsgpt.graphrag.graphrag_available", lambda: True
)
delay = _patch_delay(monkeypatch)
@@ -169,10 +169,10 @@ class TestRemoteWorkerEnqueuesGraphExtraction:
_mock_remote_pipeline,
):
"""A re-ingest clears the prior graph before re-enqueuing extraction."""
from application import worker
from docsgpt import worker
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: True
"docsgpt.graphrag.graphrag_available", lambda: True
)
reset = MagicMock(name="reset_graph")
monkeypatch.setattr(worker, "_reset_graph_for_source", reset)
@@ -201,10 +201,10 @@ class TestRemoteWorkerEnqueuesGraphExtraction:
self, task_self, pg_conn, patch_worker_db, monkeypatch,
_mock_remote_pipeline,
):
from application import worker
from docsgpt import worker
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: True
"docsgpt.graphrag.graphrag_available", lambda: True
)
delay = _patch_delay(monkeypatch)
@@ -229,10 +229,10 @@ class TestRemoteWorkerEnqueuesGraphExtraction:
self, task_self, pg_conn, patch_worker_db, monkeypatch,
_mock_remote_pipeline,
):
from application import worker
from docsgpt import worker
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: False
"docsgpt.graphrag.graphrag_available", lambda: False
)
delay = _patch_delay(monkeypatch)
@@ -261,11 +261,11 @@ class TestEnqueueIsolatesBrokerFailures:
self, pg_conn, patch_worker_db, monkeypatch
):
"""A broker hiccup in ``.delay`` must not fail an otherwise-good ingest."""
from application import worker
from application.storage.db.source_config import SourceConfig
from docsgpt import worker
from docsgpt.storage.db.source_config import SourceConfig
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: True
"docsgpt.graphrag.graphrag_available", lambda: True
)
monkeypatch.setattr(
worker, "_reset_graph_for_source", lambda *a, **kw: None
@@ -275,7 +275,7 @@ class TestEnqueueIsolatesBrokerFailures:
raise RuntimeError("broker down")
monkeypatch.setattr(
"application.api.user.tasks.extract_graph.delay", _boom
"docsgpt.api.user.tasks.extract_graph.delay", _boom
)
config = {"kind": "graphrag", "retrieval": {"retriever": "graphrag"}}
@@ -289,11 +289,11 @@ class TestEnqueueIsolatesBrokerFailures:
self, pg_conn, patch_worker_db, monkeypatch
):
"""A DB hiccup reading ``updated_at`` must also be swallowed."""
from application import worker
from application.storage.db.source_config import SourceConfig
from docsgpt import worker
from docsgpt.storage.db.source_config import SourceConfig
monkeypatch.setattr(
"application.graphrag.graphrag_available", lambda: True
"docsgpt.graphrag.graphrag_available", lambda: True
)
monkeypatch.setattr(
worker, "_reset_graph_for_source", lambda *a, **kw: None
tests/worker/test_ingest_checkpoint.py
+15 -15
View File
@@ -35,14 +35,14 @@ def patch_pipeline_db(pg_conn, monkeypatch):
yield pg_conn
monkeypatch.setattr(
"application.parser.embedding_pipeline.db_session", _use_pg_conn
"docsgpt.parser.embedding_pipeline.db_session", _use_pg_conn
)
@pytest.fixture
def faiss_settings(monkeypatch):
"""Force the embed pipeline down the faiss branch with a stub vector store."""
from application.parser import embedding_pipeline as ep
from docsgpt.parser import embedding_pipeline as ep
monkeypatch.setattr(ep.settings, "VECTOR_STORE", "faiss", raising=False)
@@ -95,7 +95,7 @@ class TestEmbedCheckpoint:
# The retry decorator wraps the real fn. Patch the module-level name
# so our loop calls the spy directly. The loop embeds a batch at a
# time, so the spy receives a list and flattens it back out.
import application.parser.embedding_pipeline as ep_mod
import docsgpt.parser.embedding_pipeline as ep_mod
original = ep_mod.add_texts_to_store_with_retry
ep_mod.add_texts_to_store_with_retry = _fake_add
@@ -127,7 +127,7 @@ class TestEmbedCheckpoint:
self, pg_conn, patch_pipeline_db, faiss_settings, tmp_path
):
"""Pre-seed progress at ``last_index=2`` and assert chunks 0..2 are not re-embedded."""
import application.parser.embedding_pipeline as ep_mod
import docsgpt.parser.embedding_pipeline as ep_mod
source_id = str(uuid.uuid4())
docs = _make_docs(6)
@@ -170,7 +170,7 @@ class TestEmbedCheckpoint:
"""Resuming a FAISS run must NOT pass ``docs_init`` (which would
overwrite the previously-saved index with a partial reset).
"""
import application.parser.embedding_pipeline as ep_mod
import docsgpt.parser.embedding_pipeline as ep_mod
monkeypatch.setattr(ep_mod.settings, "VECTOR_STORE", "faiss", raising=False)
@@ -207,7 +207,7 @@ class TestEmbedCheckpoint:
self, pg_conn, patch_pipeline_db, monkeypatch, tmp_path
):
"""Non-faiss stores must NOT have ``delete_index`` called on resume."""
import application.parser.embedding_pipeline as ep_mod
import docsgpt.parser.embedding_pipeline as ep_mod
monkeypatch.setattr(ep_mod.settings, "VECTOR_STORE", "qdrant", raising=False)
@@ -237,7 +237,7 @@ class TestEmbedCheckpoint:
"""A Celery autoretry passes the same ``self.request.id`` and
must resume from the persisted ``last_index``.
"""
import application.parser.embedding_pipeline as ep_mod
import docsgpt.parser.embedding_pipeline as ep_mod
source_id = str(uuid.uuid4())
docs = _make_docs(6)
@@ -269,7 +269,7 @@ class TestEmbedCheckpoint:
"""A fresh sync/reingest passes a new ``attempt_id`` and must
reset the checkpoint so the index is rebuilt from chunk 0.
"""
import application.parser.embedding_pipeline as ep_mod
import docsgpt.parser.embedding_pipeline as ep_mod
source_id = str(uuid.uuid4())
# Prior run completed: last_index=4 (chunks 0..4 embedded over a
@@ -320,7 +320,7 @@ class TestEmbedCheckpoint:
loop saw ``loop_start >= total_docs`` and embedded zero chunks,
leaving stale vectors in place.
"""
import application.parser.embedding_pipeline as ep_mod
import docsgpt.parser.embedding_pipeline as ep_mod
source_id = str(uuid.uuid4())
# Upload finished cleanly with 5 chunks; checkpoint reflects done.
@@ -356,7 +356,7 @@ class TestEmbedCheckpoint:
(or tests) that pass no ``attempt_id`` must still resume against
them — IS NOT DISTINCT FROM treats NULL/NULL as equal.
"""
import application.parser.embedding_pipeline as ep_mod
import docsgpt.parser.embedding_pipeline as ep_mod
source_id = str(uuid.uuid4())
_seed_progress_row(
@@ -391,7 +391,7 @@ class TestEmbedCheckpoint:
poison-loop guard finalised the row. Post-fix, the seed is
recorded immediately so ``embedded == total == 1``.
"""
import application.parser.embedding_pipeline as ep_mod
import docsgpt.parser.embedding_pipeline as ep_mod
source_id = str(uuid.uuid4())
docs = _make_docs(1)
@@ -430,7 +430,7 @@ class TestEmbedCheckpoint:
the loop's per-iteration record overshoots correctly (counts
seed + iterations) and the final state is ``embedded=total``.
"""
import application.parser.embedding_pipeline as ep_mod
import docsgpt.parser.embedding_pipeline as ep_mod
source_id = str(uuid.uuid4())
docs = _make_docs(4)
@@ -464,7 +464,7 @@ class TestIngestHeartbeat:
self, pg_conn, patch_worker_db, monkeypatch
):
"""One tick of the heartbeat must move ``last_updated`` forward."""
from application import worker
from docsgpt import worker
source_id = str(uuid.uuid4())
# Seed a row with ``last_updated`` deliberately in the past so we
@@ -506,7 +506,7 @@ class TestIngestHeartbeat:
def test_loop_swallows_db_errors(self, monkeypatch):
"""A failing DB call must not crash the daemon — it should keep ticking."""
from application import worker
from docsgpt import worker
@contextmanager
def _broken_session():
@@ -524,7 +524,7 @@ class TestIngestHeartbeat:
def test_start_and_stop_helpers_join_quickly(self, monkeypatch):
"""``_start_ingest_heartbeat`` + ``_stop_ingest_heartbeat`` must not hang."""
from application import worker
from docsgpt import worker
@contextmanager
def _noop_session():
tests/worker/test_ingest_connector.py
+7 -7
View File
@@ -1,4 +1,4 @@
"""Smoke test for ``application.worker.ingest_connector`` in sync mode.
"""Smoke test for ``docsgpt.worker.ingest_connector`` in sync mode.
Sync mode (``operation_mode="sync"``) bumps ``sources.date`` on the
target source row, the same PG side-effect as ``remote_worker``. Upload
@@ -14,14 +14,14 @@ from unittest.mock import MagicMock
import pytest
from application.parser.schema.base import Document
from application.storage.db.repositories.sources import SourcesRepository
from docsgpt.parser.schema.base import Document
from docsgpt.storage.db.repositories.sources import SourcesRepository
@pytest.fixture
def _mock_connector_pipeline(monkeypatch):
"""Stub the connector + pipeline so only PG writes are real."""
from application import worker
from docsgpt import worker
fake_connector = MagicMock(name="connector")
fake_connector.download_to_directory.return_value = {
@@ -75,7 +75,7 @@ class TestIngestConnectorSyncUpdatesDate:
monkeypatch,
_mock_connector_pipeline,
):
from application import worker
from docsgpt import worker
import datetime as dt
old_date = dt.datetime(2019, 6, 1, tzinfo=dt.timezone.utc)
@@ -122,7 +122,7 @@ class TestIngestConnectorDeterministicSourceId:
monkeypatch,
_mock_connector_pipeline,
):
from application import worker
from docsgpt import worker
captured: list[dict] = []
monkeypatch.setattr(
@@ -154,7 +154,7 @@ class TestIngestConnectorDeterministicSourceId:
monkeypatch,
_mock_connector_pipeline,
):
from application import worker
from docsgpt import worker
captured: list[dict] = []
monkeypatch.setattr(
tests/worker/test_ingest_worker.py
+11 -11
View File
@@ -1,4 +1,4 @@
"""Smoke test for ``application.worker.ingest_worker``.
"""Smoke test for ``docsgpt.worker.ingest_worker``.
``ingest_worker`` does **not** write to Postgres directly. Its PG
side-effect (creating the ``sources`` row) goes through the
@@ -22,7 +22,7 @@ from unittest.mock import MagicMock
import pytest
from application.parser.schema.base import Document
from docsgpt.parser.schema.base import Document
def _patch_ingest_pipeline(monkeypatch, captured):
@@ -34,7 +34,7 @@ def _patch_ingest_pipeline(monkeypatch, captured):
with each ``upload_index`` payload so callers can assert on the
derived ``source_id``.
"""
from application import worker
from docsgpt import worker
fake_storage = MagicMock(name="storage")
fake_storage.is_directory.return_value = False
@@ -77,7 +77,7 @@ def _spy_chunker(monkeypatch):
chunker whose ``chunk`` passes documents through unchanged, so the
config the worker threads into chunking can be asserted.
"""
from application import worker
from docsgpt import worker
calls: list[dict] = []
@@ -98,7 +98,7 @@ class TestIngestWorker:
def test_invokes_upload_index_with_expected_payload(
self, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
captured: list[dict] = []
_patch_ingest_pipeline(monkeypatch, captured)
@@ -135,7 +135,7 @@ class TestIngestWorkerConfigThreading:
def test_no_config_uses_classic_defaults(
self, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
captured: list[dict] = []
_patch_ingest_pipeline(monkeypatch, captured)
@@ -162,7 +162,7 @@ class TestIngestWorkerConfigThreading:
def test_non_default_config_is_threaded(
self, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
captured: list[dict] = []
_patch_ingest_pipeline(monkeypatch, captured)
@@ -199,7 +199,7 @@ class TestIngestWorkerDeterministicSourceId:
def test_uses_uuid5_when_idempotency_key_present(
self, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
captured: list[dict] = []
_patch_ingest_pipeline(monkeypatch, captured)
@@ -235,7 +235,7 @@ class TestIngestWorkerDeterministicSourceId:
def test_falls_back_to_uuid4_without_key(
self, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
captured: list[dict] = []
_patch_ingest_pipeline(monkeypatch, captured)
@@ -267,8 +267,8 @@ class TestIngestWorkerDeterministicSourceId:
first call and assert the second call hits the existing-row
branch instead of inserting again.
"""
from application import worker
from application.storage.db.repositories.sources import SourcesRepository
from docsgpt import worker
from docsgpt.storage.db.repositories.sources import SourcesRepository
captured: list[dict] = []
_patch_ingest_pipeline(monkeypatch, captured)
tests/worker/test_parse_document_worker.py
+7 -7
View File
@@ -1,4 +1,4 @@
"""Unit tests for ``application.worker.parse_document_worker``.
"""Unit tests for ``docsgpt.worker.parse_document_worker``.
The worker re-resolves the artifact through the run-scoped gate (independent of
the tool), reads its bytes, shapes the result, and persists when asked. The DB,
@@ -13,7 +13,7 @@ from typing import Any, Dict, Optional
import pytest
import application.worker as worker
import docsgpt.worker as worker
_ART_ID = str(uuid.uuid4())
@@ -65,7 +65,7 @@ def _patch_repo(monkeypatch, *, found: bool, run: Optional[str]):
def _patch_parse(monkeypatch, result: Dict[str, Any]):
import application.parser.document_reader as dr
import docsgpt.parser.document_reader as dr
monkeypatch.setattr(dr, "parse_document_bytes", lambda data, filename, **opts: result)
@@ -136,7 +136,7 @@ def test_persist_stores_full_result_and_returns_ref(monkeypatch):
return {"artifact_id": "new-art", "version": 1, "filename": "x.json",
"mime_type": "application/json", "size": 10}
import application.sandbox.artifacts_capture as ac
import docsgpt.sandbox.artifacts_capture as ac
monkeypatch.setattr(ac, "persist_new_artifact", _fake_persist)
@@ -158,7 +158,7 @@ def test_persist_quota_surfaces_as_artifact_error(monkeypatch):
_patch_repo(monkeypatch, found=True, run="run-1")
_patch_parse(monkeypatch, {"output": "markdown", "content": "x", "truncated": False})
import application.sandbox.artifacts_capture as ac
import docsgpt.sandbox.artifacts_capture as ac
def _quota(**kwargs):
raise ac.QuotaExceeded("artifact storage quota reached")
@@ -190,7 +190,7 @@ def test_result_payload_content_is_bounded(monkeypatch):
@pytest.mark.unit
def test_result_payload_chunks_are_bounded(monkeypatch):
import application.parser.document_reader as dr
import docsgpt.parser.document_reader as dr
_patch_repo(monkeypatch, found=True, run="run-1")
# Many oversized chunks: count is capped AND each chunk is windowed.
@@ -210,7 +210,7 @@ def test_result_payload_chunks_are_bounded(monkeypatch):
def _capture_persist(monkeypatch):
"""Patch persist_new_artifact to record what would be stored; return the capture dict."""
import application.sandbox.artifacts_capture as ac
import docsgpt.sandbox.artifacts_capture as ac
captured: Dict[str, Any] = {}
tests/worker/test_reembed_wiki_page.py
+14 -14
View File
@@ -13,8 +13,8 @@ from unittest.mock import MagicMock
import pytest
from application.parser.schema.base import Document
from application.storage.db.repositories.sources import SourcesRepository
from docsgpt.parser.schema.base import Document
from docsgpt.storage.db.repositories.sources import SourcesRepository
def _seed_source(pg_conn) -> str:
@@ -29,7 +29,7 @@ def _seed_source(pg_conn) -> str:
def _patch_store(monkeypatch, store):
monkeypatch.setattr(
"application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
"docsgpt.vectorstore.vector_creator.VectorCreator.create_vectorstore",
lambda *a, **kw: store,
)
@@ -40,7 +40,7 @@ def _patch_repo(monkeypatch, page):
repo.get_by_path.return_value = page
repo.set_embed_status.return_value = True
monkeypatch.setattr(
"application.worker.WikiPagesRepository", lambda conn: repo
"docsgpt.worker.WikiPagesRepository", lambda conn: repo
)
return repo
@@ -55,7 +55,7 @@ def _patch_chunker(monkeypatch, chunks):
return chunker
monkeypatch.setattr(
"application.worker.ChunkerCreator.create_chunker",
"docsgpt.worker.ChunkerCreator.create_chunker",
staticmethod(_create_chunker),
)
return calls
@@ -66,7 +66,7 @@ class TestReembedWikiPageWorker:
def test_page_exists_reembeds(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
@@ -107,8 +107,8 @@ class TestReembedWikiPageWorker:
check reads as the legacy model and reports as stale on every startup.
Stamping here heals a source created before it was recorded.
"""
from application import worker
from application.core.settings import settings
from docsgpt import worker
from docsgpt.core.settings import settings
source_id = _seed_source(pg_conn)
assert SourcesRepository(pg_conn).get_any(source_id, "alice")["model"] is None
@@ -128,7 +128,7 @@ class TestReembedWikiPageWorker:
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
"""Deleting a page embeds nothing, so it claims nothing about the model."""
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
_patch_store(monkeypatch, MagicMock(name="vector_store"))
@@ -143,7 +143,7 @@ class TestReembedWikiPageWorker:
def test_page_missing_purges(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
@@ -165,7 +165,7 @@ class TestReembedWikiPageWorker:
def test_embed_failure_sets_failed_and_reraises(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
source_id = _seed_source(pg_conn)
@@ -191,17 +191,17 @@ class TestReembedWikiPageWorker:
class TestReembedWikiPageTask:
def test_idempotency_key_is_content_hash(self, pg_conn, monkeypatch):
"""A redelivery with the same content_hash key short-circuits the worker."""
from application.api.user import tasks
from docsgpt.api.user import tasks
@contextmanager
def _yield():
yield pg_conn
monkeypatch.setattr(
"application.api.user.idempotency.db_session", _yield
"docsgpt.api.user.idempotency.db_session", _yield
)
monkeypatch.setattr(
"application.api.user.idempotency.db_readonly", _yield
"docsgpt.api.user.idempotency.db_readonly", _yield
)
calls: list[str] = []
tests/worker/test_reingest_source_worker.py
+7 -7
View File
@@ -1,4 +1,4 @@
"""Smoke test for ``application.worker.reingest_source_worker``.
"""Smoke test for ``docsgpt.worker.reingest_source_worker``.
The task reads a source row, diffs its stored ``directory_structure``
against what's currently in storage, updates the vector store, then
@@ -14,8 +14,8 @@ from unittest.mock import MagicMock
import pytest
from application.parser.schema.base import Document
from application.storage.db.repositories.sources import SourcesRepository
from docsgpt.parser.schema.base import Document
from docsgpt.storage.db.repositories.sources import SourcesRepository
@pytest.mark.unit
@@ -23,7 +23,7 @@ class TestReingestSourceWorker:
def test_updates_source_directory_structure_and_tokens(
self, pg_conn, patch_worker_db, task_self, monkeypatch
):
from application import worker
from docsgpt import worker
# Seed a source we can re-ingest.
src = SourcesRepository(pg_conn).create(
@@ -75,7 +75,7 @@ class TestReingestSourceWorker:
fake_store = MagicMock(name="vector_store")
fake_store.get_chunks.return_value = []
monkeypatch.setattr(
"application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
"docsgpt.vectorstore.vector_creator.VectorCreator.create_vectorstore",
lambda *a, **kw: fake_store,
)
@@ -103,7 +103,7 @@ class TestReingestSourceWorker:
then asserts ``ChunkerCreator.create_chunker`` is built with that
config — not the 1250/150 classic defaults (D1/D8).
"""
from application import worker
from docsgpt import worker
src = SourcesRepository(pg_conn).create(
"doc-set",
@@ -152,7 +152,7 @@ class TestReingestSourceWorker:
fake_store = MagicMock(name="vector_store")
fake_store.get_chunks.return_value = []
monkeypatch.setattr(
"application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
"docsgpt.vectorstore.vector_creator.VectorCreator.create_vectorstore",
lambda *a, **kw: fake_store,
)
tests/worker/test_remote_worker.py
+12 -12
View File
@@ -1,4 +1,4 @@
"""Smoke tests for ``application.worker.remote_worker`` and ``sync_worker``.
"""Smoke tests for ``docsgpt.worker.remote_worker`` and ``sync_worker``.
``remote_worker`` in ``sync`` mode does one PG write: it bumps
``sources.date`` on the referenced source row to ``now()``. That's the
@@ -19,14 +19,14 @@ from unittest.mock import MagicMock
import pytest
from application.parser.schema.base import Document
from application.storage.db.repositories.sources import SourcesRepository
from docsgpt.parser.schema.base import Document
from docsgpt.storage.db.repositories.sources import SourcesRepository
@pytest.fixture
def _mock_remote_pipeline(monkeypatch):
"""Stub out the non-PG boundaries used by ``remote_worker``."""
from application import worker
from docsgpt import worker
fake_loader = MagicMock(name="remote_loader")
fake_loader.load_data.return_value = [
@@ -59,7 +59,7 @@ class TestRemoteWorkerSyncUpdatesDate:
monkeypatch,
_mock_remote_pipeline,
):
from application import worker
from docsgpt import worker
# Seed a source with a known old ``date`` we can compare against.
import datetime as dt
@@ -111,7 +111,7 @@ class TestSyncWorker:
each to ``sync``. We assert the seeded row is discovered and
forwarded with the right doc_id — the nested ``sync`` call is
stubbed so we don't re-run the whole remote pipeline here."""
from application import worker
from docsgpt import worker
src = SourcesRepository(pg_conn).create(
"weekly-feed",
@@ -157,7 +157,7 @@ class TestSyncWorker:
):
"""connector:* sources have no RemoteCreator loader — sync_worker
must skip them, not dispatch them into sync()."""
from application import worker
from docsgpt import worker
SourcesRepository(pg_conn).create(
"drive-folder",
@@ -194,7 +194,7 @@ class TestSyncWorker:
):
"""Regression: remote_data reads back as a dict; sync_worker must
hand the loader the URL string, not the raw dict."""
from application import worker
from docsgpt import worker
SourcesRepository(pg_conn).create(
"docs-crawl",
@@ -249,7 +249,7 @@ class TestSyncWorker:
):
"""A URL source whose remote_data dict has no URL key normalizes
to None — sync_worker must skip it, not dispatch a doomed sync()."""
from application import worker
from docsgpt import worker
SourcesRepository(pg_conn).create(
"broken-feed",
@@ -292,7 +292,7 @@ class TestRemoteWorkerPathTraversal:
monkeypatch,
_mock_remote_pipeline,
):
from application import worker
from docsgpt import worker
created_paths: list[str] = []
deleted_paths: list[str] = []
@@ -350,7 +350,7 @@ class TestRemoteWorkerDeterministicSourceId:
monkeypatch,
_mock_remote_pipeline,
):
from application import worker
from docsgpt import worker
captured: list[dict] = []
monkeypatch.setattr(
@@ -382,7 +382,7 @@ class TestRemoteWorkerDeterministicSourceId:
monkeypatch,
_mock_remote_pipeline,
):
from application import worker
from docsgpt import worker
captured: list[dict] = []
monkeypatch.setattr(
tests/worker/test_worker_sse_publishes.py
+32 -32
View File
@@ -1,4 +1,4 @@
"""SSE publish wiring tests for ``application.worker``.
"""SSE publish wiring tests for ``docsgpt.worker``.
Each worker function emits ``publish_user_event`` envelopes at its
queued / progress / completed / failed boundaries. The SSE frontend's
@@ -7,7 +7,7 @@ depend on this exact emit sequence, so a regression that silently drops
a publish leaves the UI wedged on a stale "training" state until the
polling fallback rescues it.
These tests patch ``application.worker.publish_user_event`` with a
These tests patch ``docsgpt.worker.publish_user_event`` with a
capture list and assert the ordered call args per worker. Broader
worker behaviour (PG side effects, pipeline correctness) is covered by
the per-task test files in this directory; here we focus narrowly on
@@ -23,8 +23,8 @@ from unittest.mock import MagicMock
import pytest
from application.parser.schema.base import Document
from application.storage.db.repositories.sources import SourcesRepository
from docsgpt.parser.schema.base import Document
from docsgpt.storage.db.repositories.sources import SourcesRepository
# ── helpers ──────────────────────────────────────────────────────────────
@@ -61,7 +61,7 @@ class _PublishCapture:
@pytest.fixture
def publishes(monkeypatch):
"""Patch ``publish_user_event`` in the worker module and yield the capture."""
from application import worker
from docsgpt import worker
cap = _PublishCapture()
monkeypatch.setattr(worker, "publish_user_event", cap)
@@ -76,7 +76,7 @@ def _patch_ingest_pipeline_min(monkeypatch, *, raise_in_pipeline: bool = False):
``failed`` branch can be reached without otherwise rewriting the
function.
"""
from application import worker
from docsgpt import worker
fake_storage = MagicMock(name="storage")
fake_storage.is_directory.return_value = False
@@ -121,7 +121,7 @@ class TestIngestWorkerPublishes:
def test_happy_path_emits_queued_then_completed(
self, patch_worker_db, task_self, monkeypatch, publishes
):
from application import worker
from docsgpt import worker
_patch_ingest_pipeline_min(monkeypatch)
caller_source_id = str(uuid.uuid4())
@@ -168,7 +168,7 @@ class TestIngestWorkerPublishes:
def test_pipeline_failure_emits_queued_then_failed(
self, patch_worker_db, task_self, monkeypatch, publishes
):
from application import worker
from docsgpt import worker
_patch_ingest_pipeline_min(monkeypatch, raise_in_pipeline=True)
@@ -220,7 +220,7 @@ def _seed_source_for_reingest(pg_conn, *, user_id: str, name: str = "doc-set"):
def _stub_reingest_storage_and_vectorstore(monkeypatch):
from application import worker
from docsgpt import worker
fake_storage = MagicMock(name="storage")
fake_storage.is_directory.return_value = True
@@ -232,7 +232,7 @@ def _stub_reingest_storage_and_vectorstore(monkeypatch):
fake_store = MagicMock(name="vector_store")
fake_store.get_chunks.return_value = []
monkeypatch.setattr(
"application.vectorstore.vector_creator.VectorCreator.create_vectorstore",
"docsgpt.vectorstore.vector_creator.VectorCreator.create_vectorstore",
lambda *a, **kw: fake_store,
)
return fake_store
@@ -243,7 +243,7 @@ class TestReingestSourceWorkerPublishes:
def test_with_changes_emits_queued_then_completed_with_chunk_counts(
self, pg_conn, patch_worker_db, task_self, monkeypatch, publishes
):
from application import worker
from docsgpt import worker
src = _seed_source_for_reingest(pg_conn, user_id="alice")
source_id = str(src["id"])
@@ -293,7 +293,7 @@ class TestReingestSourceWorkerPublishes:
chunk-diff loop. If it didn't publish a terminal event the
toast would hang on ``training`` forever.
"""
from application import worker
from docsgpt import worker
# Seed with the same directory_structure the reader will report,
# so ``added_files`` and ``removed_files`` are both empty.
@@ -345,7 +345,7 @@ class TestReingestSourceWorkerPublishes:
toast doesn't wedge. The worker re-raises after publishing,
which is the documented contract for Celery retry handling.
"""
from application import worker
from docsgpt import worker
with pytest.raises(ValueError, match="not found"):
worker.reingest_source_worker(
@@ -370,7 +370,7 @@ class TestReingestSourceWorkerPublishes:
def _stub_remote_pipeline(monkeypatch, *, raise_in_pipeline: bool = False):
from application import worker
from docsgpt import worker
fake_loader = MagicMock(name="remote_loader")
fake_loader.load_data.return_value = [
@@ -405,7 +405,7 @@ class TestRemoteWorkerPublishes:
def test_upload_happy_path(
self, tmp_path, task_self, monkeypatch, publishes
):
from application import worker
from docsgpt import worker
_stub_remote_pipeline(monkeypatch)
@@ -438,7 +438,7 @@ class TestRemoteWorkerPublishes:
def test_upload_failure_emits_failed(
self, tmp_path, task_self, monkeypatch, publishes
):
from application import worker
from docsgpt import worker
_stub_remote_pipeline(monkeypatch, raise_in_pipeline=True)
@@ -467,7 +467,7 @@ class TestRemoteWorkerPublishes:
def _stub_connector_pipeline(
monkeypatch, *, files_downloaded: int = 1, empty_result: bool = False
):
from application import worker
from docsgpt import worker
fake_connector = MagicMock(name="connector")
fake_connector.download_to_directory.return_value = {
@@ -519,7 +519,7 @@ def _stub_connector_pipeline(
@pytest.mark.unit
class TestIngestConnectorPublishes:
def test_upload_happy_path(self, task_self, monkeypatch, publishes):
from application import worker
from docsgpt import worker
_stub_connector_pipeline(monkeypatch)
@@ -552,7 +552,7 @@ class TestIngestConnectorPublishes:
otherwise silent; without the publish the toast wedges on
``training`` until polling rescues it.
"""
from application import worker
from docsgpt import worker
_stub_connector_pipeline(
monkeypatch, files_downloaded=0, empty_result=True
@@ -586,7 +586,7 @@ class TestAttachmentWorkerPublishes:
def test_happy_path_emits_full_progress_sequence(
self, pg_conn, patch_worker_db, task_self, monkeypatch, publishes
):
from application import worker
from docsgpt import worker
fake_doc = Document(
text="hello world",
@@ -646,7 +646,7 @@ class TestAttachmentWorkerPublishes:
terminal ``failed`` must still arrive so the toast unwedges,
even though no ``current=80`` progress will follow.
"""
from application import worker
from docsgpt import worker
fake_storage = MagicMock(name="storage")
fake_storage.process_file.side_effect = RuntimeError("parse boom")
@@ -759,7 +759,7 @@ def _task_self_with_request_id():
def _patch_mcp_tool(monkeypatch, fake_tool_class) -> None:
"""Replace the ``MCPTool`` symbol the worker imports at call time.
The worker runs ``from application.agents.tools.mcp_tool import MCPTool``
The worker runs ``from docsgpt.agents.tools.mcp_tool import MCPTool``
inside the function body. Importing the real module is a circular
dependency in this test process (it pulls in the user routes,
which pull in the MCP namespace, which pulls in
@@ -771,10 +771,10 @@ def _patch_mcp_tool(monkeypatch, fake_tool_class) -> None:
import sys
import types
stub = types.ModuleType("application.agents.tools.mcp_tool")
stub = types.ModuleType("docsgpt.agents.tools.mcp_tool")
stub.MCPTool = fake_tool_class
monkeypatch.setitem(
sys.modules, "application.agents.tools.mcp_tool", stub
sys.modules, "docsgpt.agents.tools.mcp_tool", stub
)
@@ -783,7 +783,7 @@ class TestMcpOauthPublishes:
def test_happy_path_emits_progress_sequence_and_completed(
self, monkeypatch, publishes, _task_self_with_request_id
):
from application import worker
from docsgpt import worker
auth_url = "https://idp.example.com/authorize?state=happy-path"
_patch_mcp_tool(
@@ -831,7 +831,7 @@ class TestMcpOauthPublishes:
common failure mode (user cancels, provider 4xx). Must surface
as a ``mcp.oauth.failed`` envelope so the toast unwedges.
"""
from application import worker
from docsgpt import worker
_patch_mcp_tool(
monkeypatch,
@@ -864,7 +864,7 @@ class TestMcpOauthPublishes:
synthetic / shared channel. Polling-based status remains the
path of record in that case.
"""
from application import worker
from docsgpt import worker
_patch_mcp_tool(monkeypatch, _make_fake_mcp_tool_class())
@@ -902,7 +902,7 @@ class TestQueuedEventRetryGate:
def test_ingest_worker_skips_queued_on_retry(
self, patch_worker_db, task_self_retry, monkeypatch, publishes
):
from application import worker
from docsgpt import worker
_patch_ingest_pipeline_min(monkeypatch)
@@ -925,7 +925,7 @@ class TestQueuedEventRetryGate:
def test_remote_worker_skips_queued_on_retry(
self, tmp_path, task_self_retry, monkeypatch, publishes
):
from application import worker
from docsgpt import worker
_stub_remote_pipeline(monkeypatch)
@@ -944,7 +944,7 @@ class TestQueuedEventRetryGate:
def test_ingest_connector_skips_queued_on_retry(
self, task_self_retry, monkeypatch, publishes
):
from application import worker
from docsgpt import worker
_stub_connector_pipeline(monkeypatch)
@@ -964,7 +964,7 @@ class TestQueuedEventRetryGate:
def test_reingest_source_worker_skips_queued_on_retry(
self, pg_conn, patch_worker_db, task_self_retry, monkeypatch, publishes
):
from application import worker
from docsgpt import worker
src = _seed_source_for_reingest(pg_conn, user_id="alice")
source_id = str(src["id"])
@@ -996,7 +996,7 @@ class TestQueuedEventRetryGate:
"""Sanity counterpart: ``retries == 0`` keeps the original
queued+completed pair so the no-retry path is unchanged.
"""
from application import worker
from docsgpt import worker
_patch_ingest_pipeline_min(monkeypatch)