mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-13 16:14:10 +00:00
223 files changed
+23053
-5733
No files matched your search
@@ -49,6 +49,56 @@ class TestResolveSource:
|
||||
assert got is not None
|
||||
assert str(got["id"]) == str(src["id"])
|
||||
|
||||
def test_team_viewer_can_read(self, pg_conn):
|
||||
from docsgpt.api.user.sources.chunks import _resolve_source
|
||||
from docsgpt.storage.db.repositories.team_members import TeamMembersRepository
|
||||
from docsgpt.storage.db.repositories.team_resource_grants import TeamResourceGrantsRepository
|
||||
from docsgpt.storage.db.repositories.teams import TeamsRepository
|
||||
|
||||
owner, viewer = "u-resolve-owner", "u-resolve-viewer"
|
||||
src = _seed_source(pg_conn, user=owner)
|
||||
team = TeamsRepository(pg_conn).create("Acme", "acme-chunks", owner)
|
||||
TeamMembersRepository(pg_conn).add_member(team["id"], viewer, role="team_member")
|
||||
TeamResourceGrantsRepository(pg_conn).grant(
|
||||
team["id"], "source", str(src["id"]), owner_id=owner, granted_by=owner,
|
||||
access_level="viewer",
|
||||
)
|
||||
with _patch_db(pg_conn):
|
||||
got = _resolve_source(str(src["id"]), viewer)
|
||||
stranger = _resolve_source(str(src["id"]), "u-resolve-stranger")
|
||||
assert got is not None
|
||||
assert str(got["id"]) == str(src["id"])
|
||||
assert stranger is None
|
||||
|
||||
|
||||
class TestChunkMatchesPath:
|
||||
@pytest.mark.parametrize("metadata, path", [
|
||||
({"source": "/a/b/file.txt"}, "b/file.txt"),
|
||||
({"source": "file.txt"}, "file.txt"),
|
||||
({"source": "https://x.io/guides/setup", "file_path": "guides/setup.md"}, "guides/setup.md"),
|
||||
({"source": "https://github.com/o/r/blob/main/web/app.py", "title": "web/app.py"}, "web/app.py"),
|
||||
({"source": "s3://bkt/docs/a.pdf", "key": "docs/a.pdf", "title": "a.pdf"}, "docs/a.pdf"),
|
||||
# Web and Reddit chunks carry no file_path/key: the tree keys them by title.
|
||||
({"source": "https://docs.docsgpt.cloud/", "title": "Home - DocsGPT"}, "Home - DocsGPT"),
|
||||
({"source": "https://reddit.com/r/x/comments/1", "title": "A post"}, "A post"),
|
||||
])
|
||||
def test_matches(self, metadata, path):
|
||||
from docsgpt.api.user.sources.chunks import _chunk_matches_path
|
||||
assert _chunk_matches_path(metadata, path)
|
||||
|
||||
@pytest.mark.parametrize("metadata, path", [
|
||||
({"source": "/other.txt"}, "b/file.txt"),
|
||||
# Suffixes only match at a path boundary.
|
||||
({"source": "docs/data.md"}, "a.md"),
|
||||
# Title only stands in for remote chunks the tree could not key by file_path/key.
|
||||
({"source": "docs/readme.md", "title": "Readme"}, "Readme"),
|
||||
({"source": "s3://bkt/docs/a.pdf", "key": "docs/a.pdf", "title": "Report"}, "Report"),
|
||||
({"source": "https://x.io/a", "file_path": "a.md", "title": "A"}, "A"),
|
||||
])
|
||||
def test_rejects(self, metadata, path):
|
||||
from docsgpt.api.user.sources.chunks import _chunk_matches_path
|
||||
assert not _chunk_matches_path(metadata, path)
|
||||
|
||||
|
||||
class TestGetChunks:
|
||||
def test_returns_401_unauthenticated(self, app):
|
||||
@@ -146,6 +196,35 @@ class TestGetChunks:
|
||||
assert response.status_code == 200
|
||||
assert response.json["total"] == 1
|
||||
|
||||
def test_filters_title_keyed_web_page(self, app, pg_conn):
|
||||
# Single-URL sources ingested before WebLoader set ``file_path`` key
|
||||
# their only tree file by the page title; opening it must still list
|
||||
# the page's chunks without a re-ingest.
|
||||
from docsgpt.api.user.sources.chunks import GetChunks
|
||||
|
||||
user = "u-path-web"
|
||||
src = _seed_source(pg_conn, user=user)
|
||||
|
||||
fake_store = MagicMock()
|
||||
fake_store.get_chunks.return_value = [
|
||||
{"text": "home", "metadata": {
|
||||
"source": "https://docs.docsgpt.cloud/",
|
||||
"title": "Home - DocsGPT Documentation",
|
||||
}},
|
||||
]
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.api.user.sources.chunks.get_vector_store",
|
||||
return_value=fake_store,
|
||||
), app.test_request_context(
|
||||
f"/api/get_chunks?id={src['id']}&path=Home - DocsGPT Documentation"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = GetChunks().get()
|
||||
assert response.status_code == 200
|
||||
assert response.json["total"] == 1
|
||||
|
||||
def test_filters_by_search(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.chunks import GetChunks
|
||||
|
||||
@@ -523,8 +602,7 @@ class TestUpdateChunk:
|
||||
"metadata": {"title": "T"},
|
||||
}
|
||||
]
|
||||
fake_store.add_chunk.return_value = "new-chunk-id"
|
||||
fake_store.delete_chunk.return_value = True
|
||||
fake_store.update_chunk.return_value = "chunk-123"
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.api.user.sources.chunks.get_vector_store",
|
||||
@@ -541,4 +619,231 @@ class TestUpdateChunk:
|
||||
request.decoded_token = {"sub": user}
|
||||
response = UpdateChunk().put()
|
||||
assert response.status_code == 200
|
||||
assert response.json["chunk_id"] == "new-chunk-id"
|
||||
assert response.json == {
|
||||
"message": "Chunk updated successfully",
|
||||
"chunk_id": "chunk-123",
|
||||
"original_chunk_id": "chunk-123",
|
||||
}
|
||||
fake_store.update_chunk.assert_called_once()
|
||||
chunk_id, new_text, new_metadata = fake_store.update_chunk.call_args[0]
|
||||
assert (chunk_id, new_text) == ("chunk-123", "new text")
|
||||
assert new_metadata["title"] == "T"
|
||||
assert new_metadata["token_count"] > 0
|
||||
fake_store.add_chunk.assert_not_called()
|
||||
fake_store.delete_chunk.assert_not_called()
|
||||
|
||||
def test_update_failure_returns_500(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.chunks import UpdateChunk
|
||||
|
||||
user = "u-upd-fail"
|
||||
src = _seed_source(pg_conn, user=user)
|
||||
fake_store = MagicMock()
|
||||
fake_store.get_chunks.return_value = [
|
||||
{"doc_id": "chunk-123", "text": "old", "metadata": {}}
|
||||
]
|
||||
fake_store.update_chunk.side_effect = RuntimeError("embed down")
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.api.user.sources.chunks.get_vector_store",
|
||||
return_value=fake_store,
|
||||
), app.test_request_context(
|
||||
"/api/update_chunk", method="PUT",
|
||||
json={"id": str(src["id"]), "chunk_id": "chunk-123", "text": "new"},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = UpdateChunk().put()
|
||||
assert response.status_code == 500
|
||||
assert response.json == {"error": "Failed to update chunk - addition failed"}
|
||||
|
||||
|
||||
def test_invalid_metadata_keys_return_400(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.chunks import UpdateChunk
|
||||
from docsgpt.vectorstore.base import InvalidChunkMetadataError
|
||||
|
||||
user = "u-upd-bad-meta"
|
||||
src = _seed_source(pg_conn, user=user)
|
||||
fake_store = MagicMock()
|
||||
fake_store.get_chunks.return_value = [
|
||||
{"doc_id": "chunk-123", "text": "old", "metadata": {}}
|
||||
]
|
||||
fake_store.update_chunk.side_effect = InvalidChunkMetadataError(
|
||||
"Metadata key 'a.b' is not allowed"
|
||||
)
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.api.user.sources.chunks.get_vector_store",
|
||||
return_value=fake_store,
|
||||
), app.test_request_context(
|
||||
"/api/update_chunk", method="PUT",
|
||||
json={
|
||||
"id": str(src["id"]),
|
||||
"chunk_id": "chunk-123",
|
||||
"metadata": {"a.b": 1},
|
||||
},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = UpdateChunk().put()
|
||||
assert response.status_code == 400
|
||||
# Generic message; the offending key is only logged.
|
||||
assert response.json == {"error": "Invalid metadata"}
|
||||
|
||||
|
||||
class TestUpdateChunkGraphLinks:
|
||||
"""A store that re-ids an edited chunk must take the graph links with it."""
|
||||
|
||||
def _put(self, app, pg_conn, src, user, graph_store, fake_store=None):
|
||||
from docsgpt.api.user.sources.chunks import UpdateChunk
|
||||
|
||||
if fake_store is None:
|
||||
fake_store = MagicMock()
|
||||
# The base fallback: the edit comes back under a new id.
|
||||
fake_store.update_chunk.return_value = "chunk-new"
|
||||
fake_store.get_chunks.return_value = [
|
||||
{"doc_id": "chunk-old", "text": "old", "metadata": {}}
|
||||
]
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.api.user.sources.chunks.get_vector_store",
|
||||
return_value=fake_store,
|
||||
), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", return_value=graph_store
|
||||
), app.test_request_context(
|
||||
"/api/update_chunk", method="PUT",
|
||||
json={"id": str(src["id"]), "chunk_id": "chunk-old", "text": "new"},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
return UpdateChunk().put()
|
||||
|
||||
def _graph_source(self, pg_conn, user):
|
||||
from docsgpt.storage.db.repositories.sources import SourcesRepository
|
||||
from docsgpt.storage.db.source_config import SourceConfig
|
||||
|
||||
return SourcesRepository(pg_conn).create(
|
||||
"g", user_id=user, config=SourceConfig.parse({}).graph_enabled()
|
||||
)
|
||||
|
||||
def test_graphrag_source_remaps_links_to_the_new_chunk(self, app, pg_conn):
|
||||
user = "u-upd-graph"
|
||||
src = self._graph_source(pg_conn, user)
|
||||
graph_store = MagicMock()
|
||||
|
||||
response = self._put(app, pg_conn, src, user, graph_store)
|
||||
|
||||
assert response.status_code == 200
|
||||
graph_store.remap_chunk.assert_called_once_with(
|
||||
str(src["id"]), "chunk-old", "chunk-new"
|
||||
)
|
||||
|
||||
def test_classic_source_skips_the_graph(self, app, pg_conn):
|
||||
user = "u-upd-classic"
|
||||
src = _seed_source(pg_conn, user=user)
|
||||
graph_store = MagicMock()
|
||||
|
||||
response = self._put(app, pg_conn, src, user, graph_store)
|
||||
|
||||
assert response.status_code == 200
|
||||
graph_store.remap_chunk.assert_not_called()
|
||||
|
||||
def test_remap_failure_keeps_the_saved_edit(self, app, pg_conn):
|
||||
user = "u-upd-graph-fail"
|
||||
src = self._graph_source(pg_conn, user)
|
||||
graph_store = MagicMock()
|
||||
graph_store.remap_chunk.side_effect = RuntimeError("boom")
|
||||
|
||||
response = self._put(app, pg_conn, src, user, graph_store)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["chunk_id"] == "chunk-new"
|
||||
|
||||
def test_in_place_update_skips_the_graph_remap(self, app, pg_conn):
|
||||
user = "u-upd-graph-inplace"
|
||||
src = self._graph_source(pg_conn, user)
|
||||
graph_store = MagicMock()
|
||||
fake_store = MagicMock()
|
||||
fake_store.update_chunk.return_value = "chunk-old"
|
||||
|
||||
response = self._put(app, pg_conn, src, user, graph_store, fake_store)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["chunk_id"] == "chunk-old"
|
||||
assert response.json["original_chunk_id"] == "chunk-old"
|
||||
graph_store.remap_chunk.assert_not_called()
|
||||
|
||||
def test_default_fallback_store_remaps_to_the_new_id(self, app, pg_conn):
|
||||
"""A store without its own update (e.g. Milvus) re-adds and re-ids."""
|
||||
from docsgpt.vectorstore.base import BaseVectorStore
|
||||
|
||||
class _FallbackStore(BaseVectorStore):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.deleted = []
|
||||
|
||||
def search(self, *args, **kwargs):
|
||||
return []
|
||||
|
||||
def add_texts(self, texts, metadatas=None, *args, **kwargs):
|
||||
return []
|
||||
|
||||
def get_chunks(self):
|
||||
return [{"doc_id": "chunk-old", "text": "old", "metadata": {}}]
|
||||
|
||||
def add_chunk(self, text, metadata=None):
|
||||
return "chunk-new"
|
||||
|
||||
def delete_chunk(self, chunk_id):
|
||||
self.deleted.append(chunk_id)
|
||||
return True
|
||||
|
||||
user = "u-upd-graph-fallback"
|
||||
src = self._graph_source(pg_conn, user)
|
||||
graph_store = MagicMock()
|
||||
store = _FallbackStore()
|
||||
store_proxy = MagicMock(wraps=store)
|
||||
|
||||
response = self._put(app, pg_conn, src, user, graph_store, store_proxy)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["chunk_id"] == "chunk-new"
|
||||
assert store.deleted == ["chunk-old"]
|
||||
graph_store.remap_chunk.assert_called_once_with(
|
||||
str(src["id"]), "chunk-old", "chunk-new"
|
||||
)
|
||||
|
||||
def test_default_fallback_failed_delete_returns_500_without_duplicate(self, app, pg_conn):
|
||||
"""A failed old-chunk delete rolls the new chunk back and fails the request."""
|
||||
from docsgpt.vectorstore.base import BaseVectorStore
|
||||
|
||||
class _FallbackStore(BaseVectorStore):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.deleted = []
|
||||
|
||||
def search(self, *args, **kwargs):
|
||||
return []
|
||||
|
||||
def add_texts(self, texts, metadatas=None, *args, **kwargs):
|
||||
return []
|
||||
|
||||
def get_chunks(self):
|
||||
return [{"doc_id": "chunk-old", "text": "old", "metadata": {}}]
|
||||
|
||||
def add_chunk(self, text, metadata=None):
|
||||
return "chunk-new"
|
||||
|
||||
def delete_chunk(self, chunk_id):
|
||||
self.deleted.append(chunk_id)
|
||||
return chunk_id != "chunk-old"
|
||||
|
||||
user = "u-upd-graph-fallback-fail"
|
||||
src = self._graph_source(pg_conn, user)
|
||||
graph_store = MagicMock()
|
||||
store = _FallbackStore()
|
||||
store_proxy = MagicMock(wraps=store)
|
||||
|
||||
response = self._put(app, pg_conn, src, user, graph_store, store_proxy)
|
||||
|
||||
assert response.status_code == 500
|
||||
assert store.deleted == ["chunk-old", "chunk-new"]
|
||||
graph_store.remap_chunk.assert_not_called()
|
||||
@@ -89,6 +89,7 @@ class TestSourceGraph:
|
||||
|
||||
store = MagicMock()
|
||||
store.count_nodes.return_value = 3
|
||||
store.count_edges.return_value = 5
|
||||
store.get_graph_overview.return_value = {
|
||||
"nodes": [
|
||||
{"id": "n1", "name": "A", "type": "person",
|
||||
@@ -114,6 +115,9 @@ class TestSourceGraph:
|
||||
assert response.json["success"] is True
|
||||
assert {n["id"] for n in response.json["nodes"]} == {"n1", "n2"}
|
||||
assert response.json["edges"][0]["source"] == "n1"
|
||||
# Totals describe the whole graph, not the bounded overview.
|
||||
assert response.json["stats"] == {"nodes": 3, "edges": 5}
|
||||
store.count_edges.assert_called_once_with(sid)
|
||||
# The store receives the source's resolved id and the clamped limit.
|
||||
args = store.get_graph_overview.call_args.args
|
||||
assert args[0] == sid
|
||||
@@ -140,6 +144,7 @@ class TestSourceGraph:
|
||||
assert response.status_code == 200
|
||||
assert response.json["nodes"] == []
|
||||
assert response.json["edges"] == []
|
||||
assert response.json["stats"] == {"nodes": 0, "edges": 0}
|
||||
# No graph rows → never query the overview.
|
||||
store.get_graph_overview.assert_not_called()
|
||||
|
||||
@@ -172,6 +177,7 @@ class TestSourceGraph:
|
||||
|
||||
store = MagicMock()
|
||||
store.count_nodes.return_value = 1
|
||||
store.count_edges.return_value = 0
|
||||
store.get_graph_overview.return_value = {
|
||||
"nodes": [
|
||||
{"id": "n1", "name": "A", "type": None,
|
||||
@@ -189,8 +195,336 @@ class TestSourceGraph:
|
||||
|
||||
assert response.status_code == 200
|
||||
assert {n["id"] for n in response.json["nodes"]} == {"n1"}
|
||||
assert response.json["stats"] == {"nodes": 1, "edges": 0}
|
||||
|
||||
|
||||
def _node_list_store():
|
||||
store = MagicMock()
|
||||
store.list_nodes.return_value = {
|
||||
"nodes": [
|
||||
{"id": "n1", "name": "Ada", "type": "Person", "degree": 3,
|
||||
"doc_freq": 2},
|
||||
],
|
||||
"total": 41,
|
||||
}
|
||||
store.node_type_facets.return_value = [
|
||||
{"key": "person", "label": "Person", "count": 30},
|
||||
{"key": "", "label": None, "count": 11},
|
||||
]
|
||||
return store
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSourceGraphNodes:
|
||||
def test_returns_401_unauthenticated(self, app):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
with app.test_request_context("/api/sources/x/graph/nodes"):
|
||||
from flask import request
|
||||
request.decoded_token = None
|
||||
response = SourceGraphNodes().get("x")
|
||||
assert response.status_code == 401
|
||||
|
||||
def test_owner_gets_paged_nodes_and_type_facets(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
user = "u-graph-nodes-owner"
|
||||
sid = _graphrag_source(pg_conn, user)
|
||||
store = _node_list_store()
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", return_value=store
|
||||
), app.test_request_context(f"/api/sources/{sid}/graph/nodes"):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceGraphNodes().get(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json == {
|
||||
"success": True,
|
||||
"nodes": [
|
||||
{"id": "n1", "name": "Ada", "type": "Person", "degree": 3,
|
||||
"doc_freq": 2},
|
||||
],
|
||||
"total": 41,
|
||||
"page": 1,
|
||||
"per_page": 25,
|
||||
"types": [
|
||||
{"key": "person", "label": "Person", "count": 30},
|
||||
{"key": "", "label": None, "count": 11},
|
||||
],
|
||||
}
|
||||
store.list_nodes.assert_called_once_with(
|
||||
sid, query=None, type_key=None, offset=0, limit=25
|
||||
)
|
||||
store.node_type_facets.assert_called_once_with(sid)
|
||||
|
||||
def test_forwards_filters_and_clamps_paging(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
user = "u-graph-nodes-params"
|
||||
sid = _graphrag_source(pg_conn, user)
|
||||
store = _node_list_store()
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", return_value=store
|
||||
), app.test_request_context(
|
||||
f"/api/sources/{sid}/graph/nodes?q=%20ada%20&type=person&page=3&per_page=500"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceGraphNodes().get(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["page"] == 3
|
||||
assert response.json["per_page"] == 100
|
||||
store.list_nodes.assert_called_once_with(
|
||||
sid, query="ada", type_key="person", offset=200, limit=100
|
||||
)
|
||||
|
||||
def test_caps_search_query_length(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
user = "u-graph-nodes-long-q"
|
||||
sid = _graphrag_source(pg_conn, user)
|
||||
store = _node_list_store()
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", return_value=store
|
||||
), app.test_request_context(
|
||||
f"/api/sources/{sid}/graph/nodes?q=%20{'a' * 500}"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceGraphNodes().get(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
store.list_nodes.assert_called_once_with(
|
||||
sid, query="a" * 200, type_key=None, offset=0, limit=25
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"qs, page, per_page",
|
||||
[
|
||||
("page=0&per_page=0", 1, 1),
|
||||
("page=-4&per_page=-9", 1, 1),
|
||||
("page=abc&per_page=xyz", 1, 25),
|
||||
],
|
||||
)
|
||||
def test_bad_paging_values_fall_back(self, app, pg_conn, qs, page, per_page):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
user = "u-graph-nodes-bad-paging"
|
||||
sid = _graphrag_source(pg_conn, user)
|
||||
store = _node_list_store()
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", return_value=store
|
||||
), app.test_request_context(f"/api/sources/{sid}/graph/nodes?{qs}"):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceGraphNodes().get(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert (response.json["page"], response.json["per_page"]) == (page, per_page)
|
||||
kwargs = store.list_nodes.call_args.kwargs
|
||||
assert kwargs["offset"] == (page - 1) * per_page
|
||||
assert kwargs["limit"] == per_page
|
||||
|
||||
def test_non_owner_without_grant_404(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
owner = "u-graph-nodes-owner2"
|
||||
stranger = "u-graph-nodes-stranger"
|
||||
sid = _graphrag_source(pg_conn, owner)
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore"
|
||||
) as mock_store, app.test_request_context(
|
||||
f"/api/sources/{sid}/graph/nodes"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": stranger}
|
||||
response = SourceGraphNodes().get(sid)
|
||||
|
||||
assert response.status_code == 404
|
||||
mock_store.assert_not_called()
|
||||
|
||||
def test_team_viewer_can_read(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
owner = "alice-graph-nodes"
|
||||
viewer = "bob-graph-nodes-viewer"
|
||||
sid = _graphrag_source(pg_conn, owner)
|
||||
_grant_team_access(pg_conn, owner, viewer, sid, "viewer")
|
||||
store = _node_list_store()
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", return_value=store
|
||||
), app.test_request_context(f"/api/sources/{sid}/graph/nodes"):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": viewer}
|
||||
response = SourceGraphNodes().get(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["total"] == 41
|
||||
assert store.list_nodes.call_args.args == (sid,)
|
||||
|
||||
def test_store_failure_returns_400(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
user = "u-graph-nodes-fail"
|
||||
sid = _graphrag_source(pg_conn, user)
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", side_effect=RuntimeError("no db")
|
||||
), app.test_request_context(f"/api/sources/{sid}/graph/nodes"):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceGraphNodes().get(sid)
|
||||
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def _real_store_with_mock_conn(fail: bool = False, total: int = 0):
|
||||
"""A real ``GraphStore`` whose connection is a mock, so the store's own
|
||||
error handling runs. ``fail`` makes every query raise."""
|
||||
from docsgpt.graphrag.store import GraphStore
|
||||
|
||||
store = GraphStore.__new__(GraphStore)
|
||||
cursor = MagicMock()
|
||||
cursor.fetchone.return_value = [total]
|
||||
cursor.fetchall.return_value = []
|
||||
if fail:
|
||||
cursor.execute.side_effect = RuntimeError("connection reset")
|
||||
conn = MagicMock()
|
||||
conn.cursor.return_value = cursor
|
||||
store._connection = conn
|
||||
store._get_connection = lambda: conn
|
||||
store._tables_ensured = True
|
||||
return store, cursor
|
||||
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSourceGraphNodesRealStore:
|
||||
def test_query_failure_returns_400_not_empty_200(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
user = "u-graph-nodes-query-fail"
|
||||
sid = _graphrag_source(pg_conn, user)
|
||||
store, _ = _real_store_with_mock_conn(fail=True)
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", return_value=store
|
||||
), app.test_request_context(f"/api/sources/{sid}/graph/nodes"):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceGraphNodes().get(sid)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json == {"success": False}
|
||||
|
||||
def test_empty_graph_returns_empty_200(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
user = "u-graph-nodes-empty"
|
||||
sid = _graphrag_source(pg_conn, user)
|
||||
store, _ = _real_store_with_mock_conn(total=0)
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", return_value=store
|
||||
), app.test_request_context(f"/api/sources/{sid}/graph/nodes"):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceGraphNodes().get(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["nodes"] == []
|
||||
assert response.json["total"] == 0
|
||||
assert response.json["types"] == []
|
||||
|
||||
def _get(self, app, pg_conn, user, store=None, store_error=None, qs=""):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
|
||||
sid = _graphrag_source(pg_conn, user)
|
||||
kwargs = {"side_effect": store_error} if store_error else {"return_value": store}
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", **kwargs
|
||||
), app.test_request_context(f"/api/sources/{sid}/graph/nodes{qs}"):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
return SourceGraphNodes().get(sid)
|
||||
|
||||
def test_missing_graph_tables_return_empty_200(self, app, pg_conn):
|
||||
import psycopg
|
||||
|
||||
store, cursor = _real_store_with_mock_conn()
|
||||
cursor.execute.side_effect = psycopg.errors.UndefinedTable(
|
||||
'relation "graph_nodes" does not exist'
|
||||
)
|
||||
|
||||
response = self._get(
|
||||
app, pg_conn, "u-graph-nodes-no-tables", store=store,
|
||||
qs="?type=person&page=2&per_page=10",
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json == {
|
||||
"success": True,
|
||||
"nodes": [],
|
||||
"total": 0,
|
||||
"page": 2,
|
||||
"per_page": 10,
|
||||
"types": [],
|
||||
}
|
||||
|
||||
def test_unconfigured_graph_store_returns_empty_200(self, app, pg_conn):
|
||||
response = self._get(
|
||||
app, pg_conn, "u-graph-nodes-unconfigured",
|
||||
store_error=ValueError("PostgreSQL connection string is required."),
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["nodes"] == []
|
||||
assert response.json["total"] == 0
|
||||
assert response.json["types"] == []
|
||||
|
||||
def test_other_store_errors_still_return_400(self, app, pg_conn):
|
||||
import psycopg
|
||||
|
||||
store, cursor = _real_store_with_mock_conn()
|
||||
cursor.execute.side_effect = psycopg.errors.InvalidTextRepresentation("bad")
|
||||
|
||||
response = self._get(app, pg_conn, "u-graph-nodes-bad-input", store=store)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json == {"success": False}
|
||||
|
||||
def test_huge_page_is_clamped_and_keeps_the_total(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import SourceGraphNodes
|
||||
from docsgpt.graphrag.store import GRAPH_NODE_LIST_MAX_PAGE
|
||||
|
||||
user = "u-graph-nodes-huge-page"
|
||||
sid = _graphrag_source(pg_conn, user)
|
||||
store, cursor = _real_store_with_mock_conn(total=41)
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"docsgpt.graphrag.store.GraphStore", return_value=store
|
||||
), app.test_request_context(
|
||||
f"/api/sources/{sid}/graph/nodes?page=100000000000000000000&per_page=100"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceGraphNodes().get(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["total"] == 41
|
||||
assert response.json["nodes"] == []
|
||||
assert response.json["page"] == GRAPH_NODE_LIST_MAX_PAGE
|
||||
offset = cursor.execute.call_args_list[1].args[1][-1]
|
||||
assert offset == (GRAPH_NODE_LIST_MAX_PAGE - 1) * 100
|
||||
assert offset < 2**63
|
||||
|
||||
@pytest.mark.unit
|
||||
class TestSourceGraphNode:
|
||||
def test_returns_401_unauthenticated(self, app):
|
||||
@@ -216,6 +550,11 @@ class TestSourceGraphNode:
|
||||
"description": "A mathematician.",
|
||||
"degree": 3,
|
||||
"doc_freq": 2,
|
||||
"relationships": [
|
||||
{"id": "n2", "name": "Alan", "type": "person", "degree": 2,
|
||||
"edge_type": "knows", "direction": "out"},
|
||||
],
|
||||
"relationships_total": 1204,
|
||||
"chunks": [{"chunk_id": "5", "text": "body", "metadata": {}}],
|
||||
}
|
||||
|
||||
@@ -229,6 +568,9 @@ class TestSourceGraphNode:
|
||||
assert response.status_code == 200
|
||||
assert response.json["node"]["name"] == "Ada"
|
||||
assert response.json["node"]["chunks"][0]["text"] == "body"
|
||||
# The capped list and its true total pass through untouched.
|
||||
assert len(response.json["node"]["relationships"]) == 1
|
||||
assert response.json["node"]["relationships_total"] == 1204
|
||||
store.get_node_detail.assert_called_once_with(sid, "n1")
|
||||
|
||||
def test_unknown_node_404(self, app, pg_conn):
|
||||
|
||||
@@ -887,6 +887,49 @@ class TestDirectoryStructure:
|
||||
assert data["provider"] == "gdrive"
|
||||
assert data["base_path"] == "/data/nested"
|
||||
|
||||
def test_team_viewer_can_read(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import DirectoryStructure
|
||||
from docsgpt.storage.db.repositories.team_members import TeamMembersRepository
|
||||
from docsgpt.storage.db.repositories.team_resource_grants import TeamResourceGrantsRepository
|
||||
from docsgpt.storage.db.repositories.teams import TeamsRepository
|
||||
|
||||
owner, viewer = "u-dir-owner", "u-dir-viewer"
|
||||
src = _seed_source(
|
||||
pg_conn, owner, name="shared",
|
||||
directory_structure={"a.docx": {"type": "application/docx"}},
|
||||
)
|
||||
team = TeamsRepository(pg_conn).create("Acme", "acme-dir", owner)
|
||||
TeamMembersRepository(pg_conn).add_member(team["id"], viewer, role="team_member")
|
||||
TeamResourceGrantsRepository(pg_conn).grant(
|
||||
team["id"], "source", str(src["id"]), owner_id=owner, granted_by=owner,
|
||||
access_level="viewer",
|
||||
)
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/directory_structure?id={src['id']}"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": viewer}
|
||||
response = DirectoryStructure().get()
|
||||
assert response.status_code == 200
|
||||
assert response.json["directory_structure"] == {"a.docx": {"type": "application/docx"}}
|
||||
|
||||
def test_non_owner_without_grant_404(self, app, pg_conn):
|
||||
from docsgpt.api.user.sources.routes import DirectoryStructure
|
||||
|
||||
src = _seed_source(
|
||||
pg_conn, "u-dir-owner2", name="private",
|
||||
directory_structure={"a.txt": None},
|
||||
)
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/directory_structure?id={src['id']}"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": "u-dir-stranger"}
|
||||
response = DirectoryStructure().get()
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
class TestSourceConfigResource:
|
||||
def test_returns_401_unauthenticated(self, app):
|
||||
|
||||
@@ -506,3 +506,136 @@ class TestUnexpectedExceptionMasked:
|
||||
assert body["message"] == "internal error"
|
||||
# Defensive: the internal-sounding detail must not leak through.
|
||||
assert "secret connection string" not in resp.get_data(as_text=True)
|
||||
|
||||
|
||||
class TestAgentScheduleStats:
|
||||
def _get(self, app, pg_conn, agent_id: str, *, user="u1", query=""):
|
||||
from docsgpt.api.user.schedules.routes import AgentScheduleStats
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/agents/{agent_id}/schedules/stats{query}", method="GET",
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user} if user else None
|
||||
return AgentScheduleStats().get(agent_id)
|
||||
|
||||
def _run(self, conn, schedule_id, user_id, agent_id, *, ago, **fields):
|
||||
from docsgpt.storage.db.repositories.schedule_runs import (
|
||||
ScheduleRunsRepository,
|
||||
)
|
||||
|
||||
repo = ScheduleRunsRepository(conn)
|
||||
run = repo.record_pending(schedule_id, user_id, agent_id, _now() - ago)
|
||||
return repo.update(str(run["id"]), fields)
|
||||
|
||||
def test_unauthorized(self, app):
|
||||
resp = self._get(app, None, "x", user=None)
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_agent_not_found(self, app, pg_conn):
|
||||
resp = self._get(
|
||||
app, pg_conn, "00000000-0000-0000-0000-000000000000",
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
assert resp.get_json()["message"] == "agent not found"
|
||||
|
||||
def test_other_users_agent_not_found(self, app, pg_conn):
|
||||
agent_id = _make_agent(pg_conn, user_id="u2")
|
||||
resp = self._get(app, pg_conn, agent_id)
|
||||
assert resp.status_code == 404
|
||||
|
||||
def test_empty_window(self, app, pg_conn):
|
||||
agent_id = _make_agent(pg_conn)
|
||||
resp = self._get(app, pg_conn, agent_id)
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json() == {
|
||||
"days": 30, "runs": 0, "failed": 0, "tokens": 0,
|
||||
"latest_failure": None,
|
||||
}
|
||||
|
||||
def test_counts_and_latest_failure(self, app, pg_conn):
|
||||
agent_id = _make_agent(pg_conn)
|
||||
s = SchedulesRepository(pg_conn).create(
|
||||
user_id="u1", agent_id=agent_id, trigger_type="recurring",
|
||||
instruction="i", cron="* * * * *",
|
||||
next_run_at=_now() + timedelta(hours=1),
|
||||
)
|
||||
sid = str(s["id"])
|
||||
self._run(
|
||||
pg_conn, sid, "u1", agent_id, ago=timedelta(days=1),
|
||||
status="success", prompt_tokens=100, generated_tokens=50,
|
||||
)
|
||||
self._run(
|
||||
pg_conn, sid, "u1", agent_id, ago=timedelta(days=5),
|
||||
status="failed", prompt_tokens=10, generated_tokens=5,
|
||||
error_type="agent_error",
|
||||
)
|
||||
latest = self._run(
|
||||
pg_conn, sid, "u1", agent_id, ago=timedelta(days=2),
|
||||
status="timeout", prompt_tokens=1, generated_tokens=2,
|
||||
error_type="timeout",
|
||||
)
|
||||
self._run(
|
||||
pg_conn, sid, "u1", agent_id, ago=timedelta(days=45),
|
||||
status="failed", prompt_tokens=999, generated_tokens=999,
|
||||
)
|
||||
other = SchedulesRepository(pg_conn).create(
|
||||
user_id="u2", agent_id=agent_id, trigger_type="recurring",
|
||||
instruction="i", cron="* * * * *",
|
||||
next_run_at=_now() + timedelta(hours=1),
|
||||
)
|
||||
self._run(
|
||||
pg_conn, str(other["id"]), "u2", agent_id, ago=timedelta(hours=1),
|
||||
status="failed", prompt_tokens=7, generated_tokens=7,
|
||||
)
|
||||
|
||||
resp = self._get(app, pg_conn, agent_id)
|
||||
assert resp.status_code == 200
|
||||
body = resp.get_json()
|
||||
assert body["days"] == 30
|
||||
assert body["runs"] == 3
|
||||
assert body["failed"] == 2
|
||||
assert body["tokens"] == 168
|
||||
assert body["latest_failure"] == {
|
||||
"scheduled_for": latest["scheduled_for"],
|
||||
"status": "timeout",
|
||||
"error_type": "timeout",
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query, expected",
|
||||
[("?days=0", 1), ("?days=-5", 1), ("?days=9999", 365),
|
||||
("?days=7", 7), ("?days=abc", 30), ("", 30)],
|
||||
)
|
||||
def test_days_clamped(self, app, pg_conn, query, expected):
|
||||
agent_id = _make_agent(pg_conn)
|
||||
resp = self._get(app, pg_conn, agent_id, query=query)
|
||||
assert resp.status_code == 200
|
||||
assert resp.get_json()["days"] == expected
|
||||
|
||||
def test_db_error_returns_500(self, app, pg_conn):
|
||||
from docsgpt.storage.db.repositories.schedule_runs import (
|
||||
ScheduleRunsRepository,
|
||||
)
|
||||
|
||||
agent_id = _make_agent(pg_conn)
|
||||
with patch.object(
|
||||
ScheduleRunsRepository, "stats_for_agent",
|
||||
side_effect=RuntimeError("secret detail"), create=True,
|
||||
):
|
||||
resp = self._get(app, pg_conn, agent_id)
|
||||
assert resp.status_code == 500
|
||||
assert "secret detail" not in resp.get_data(as_text=True)
|
||||
|
||||
def test_route_registered_without_collision(self):
|
||||
from docsgpt.api.user.schedules.routes import (
|
||||
AgentScheduleStats,
|
||||
schedules_ns,
|
||||
)
|
||||
|
||||
paths = {
|
||||
r.urls[0]: r.resource for r in schedules_ns.resources
|
||||
}
|
||||
assert paths["/agents/<string:agent_id>/schedules/stats"] is (
|
||||
AgentScheduleStats
|
||||
)
|
||||
Reference in new issue
Block a user