mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-15 20:15:30 +00:00
feat(wiki): convert existing source to wiki + human page edit endpoint
Unit 5 of F-Wiki (D20-D23). convert_source_to_wiki task reuses reingest's storage file-load + parser to materialize files->wiki_pages (one page/file), skips/reports non-text, re-embeds per page, and flips kind=wiki + exposure=agentic_tool only when pages were created. POST /wiki/convert (explicit, write-authz; blank source enables inline, fileful enqueues the task; rejects mid-ingest). PUT /wiki/page for human edits (write-authz, optimistic version -> 409, re-embed). PATCH /config preserves kind (kind changes only via convert).
This commit is contained in:
1 parent
5110189143
commit
0c9e0313bd
8 files changed
+1133
-25
No files matched your search
@@ -1029,3 +1029,108 @@ class TestSourceConfigResource:
|
||||
# owner-scoped read sees it.
|
||||
got = SourcesRepository(pg_conn).get_any(sid, owner)
|
||||
assert got["config"]["retrieval"]["chunks"] == 9
|
||||
|
||||
def test_kind_flip_to_wiki_rejected(self, app, pg_conn):
|
||||
# Flipping kind to wiki must route through /wiki/convert, not config.
|
||||
from application.api.user.sources.routes import SourceConfigResource
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
user = "u-cfg-wiki-flip"
|
||||
src = _seed_source(pg_conn, user, name="cfg-wiki", type="file")
|
||||
sid = str(src["id"])
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/config",
|
||||
method="PATCH",
|
||||
json={"kind": "wiki"},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceConfigResource().patch(sid)
|
||||
|
||||
assert response.status_code == 400
|
||||
# The kind must NOT have silently flipped.
|
||||
from application.storage.db.source_config import SourceConfig
|
||||
|
||||
got = SourcesRepository(pg_conn).get_any(sid, user)
|
||||
assert SourceConfig.parse(got.get("config")).kind != "wiki"
|
||||
|
||||
def test_other_edits_work_on_wiki_source(self, app, pg_conn):
|
||||
# A wiki source can still edit retrieval (kind stays wiki, no reject).
|
||||
from application.api.user.sources.routes import SourceConfigResource
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
user = "u-cfg-wiki-edit"
|
||||
src = _seed_source(
|
||||
pg_conn, user, name="wiki-cfg", type="wiki",
|
||||
config={"kind": "wiki"},
|
||||
)
|
||||
sid = str(src["id"])
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/config",
|
||||
method="PATCH",
|
||||
json={"kind": "wiki", "retrieval": {"chunks": 4}},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceConfigResource().patch(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
got = SourcesRepository(pg_conn).get_any(sid, user)
|
||||
assert got["config"]["retrieval"]["chunks"] == 4
|
||||
|
||||
def test_partial_edit_preserves_wiki_kind(self, app, pg_conn):
|
||||
# A partial edit that OMITS kind must not demote a wiki to classic
|
||||
# (SourceConfig.kind defaults to "classic" on a full-replace write).
|
||||
from application.api.user.sources.routes import SourceConfigResource
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
from application.storage.db.source_config import SourceConfig
|
||||
|
||||
user = "u-cfg-wiki-partial"
|
||||
src = _seed_source(
|
||||
pg_conn, user, name="wiki-partial", type="wiki",
|
||||
config={"kind": "wiki"},
|
||||
)
|
||||
sid = str(src["id"])
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/config",
|
||||
method="PATCH",
|
||||
json={"retrieval": {"chunks": 6}},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceConfigResource().patch(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["config"]["kind"] == "wiki"
|
||||
got = SourcesRepository(pg_conn).get_any(sid, user)
|
||||
assert SourceConfig.parse(got.get("config")).kind == "wiki"
|
||||
assert got["config"]["retrieval"]["chunks"] == 6
|
||||
|
||||
def test_explicit_kind_demotion_from_wiki_rejected(self, app, pg_conn):
|
||||
# Demoting wiki -> classic via config is rejected; use /wiki/convert.
|
||||
from application.api.user.sources.routes import SourceConfigResource
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
from application.storage.db.source_config import SourceConfig
|
||||
|
||||
user = "u-cfg-wiki-demote"
|
||||
src = _seed_source(
|
||||
pg_conn, user, name="wiki-demote", type="wiki",
|
||||
config={"kind": "wiki"},
|
||||
)
|
||||
sid = str(src["id"])
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/config",
|
||||
method="PATCH",
|
||||
json={"kind": "classic"},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = SourceConfigResource().patch(sid)
|
||||
|
||||
assert response.status_code == 400
|
||||
got = SourcesRepository(pg_conn).get_any(sid, user)
|
||||
assert SourceConfig.parse(got.get("config")).kind == "wiki"
|
||||
@@ -290,3 +290,276 @@ class TestWikiPage:
|
||||
request.decoded_token = {"sub": stranger}
|
||||
response = WikiPage().get(sid)
|
||||
assert response.status_code == 404
|
||||
|
||||
|
||||
class TestConvertSourceToWiki:
|
||||
def test_returns_401_unauthenticated(self, app):
|
||||
from application.api.user.sources.routes import ConvertSourceToWiki
|
||||
|
||||
with app.test_request_context(
|
||||
"/api/sources/x/wiki/convert", method="POST"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = None
|
||||
response = ConvertSourceToWiki().post("x")
|
||||
assert response.status_code == 401
|
||||
|
||||
def test_blank_source_enabled_inline_no_task(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import ConvertSourceToWiki
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
from application.storage.db.source_config import SourceConfig
|
||||
|
||||
user = "u-convert-blank"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"blank", user_id=user, type="file", directory_structure={}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.convert_source_to_wiki.delay"
|
||||
) as mock_convert, app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/convert", method="POST"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = ConvertSourceToWiki().post(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["converted"] is False
|
||||
assert response.json["enabled"] is True
|
||||
mock_convert.assert_not_called()
|
||||
got = SourcesRepository(pg_conn).get_any(sid, user)
|
||||
cfg = SourceConfig.parse(got.get("config"))
|
||||
assert cfg.kind == "wiki"
|
||||
assert cfg.retrieval.exposure == "agentic_tool"
|
||||
|
||||
def test_fileful_source_enqueues_task(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import ConvertSourceToWiki
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
user = "u-convert-files"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"files", user_id=user, type="file",
|
||||
directory_structure={"a.md": {"type": "text/markdown"}},
|
||||
)
|
||||
sid = str(src["id"])
|
||||
|
||||
fake_task = type("T", (), {"id": "task-xyz"})()
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.convert_source_to_wiki.delay",
|
||||
return_value=fake_task,
|
||||
) as mock_convert, app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/convert", method="POST"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = ConvertSourceToWiki().post(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["task_id"] == "task-xyz"
|
||||
mock_convert.assert_called_once()
|
||||
assert mock_convert.call_args.kwargs["source_id"] == sid
|
||||
assert mock_convert.call_args.kwargs["user"] == user
|
||||
assert mock_convert.call_args.kwargs["idempotency_key"] == (
|
||||
f"convert-wiki:{sid}"
|
||||
)
|
||||
|
||||
def test_in_progress_ingest_rejected_409(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import ConvertSourceToWiki
|
||||
from application.storage.db.repositories.ingest_chunk_progress import (
|
||||
IngestChunkProgressRepository,
|
||||
)
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
user = "u-convert-ingesting"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"ingesting", user_id=user, type="file",
|
||||
directory_structure={"a.md": {"type": "text/markdown"}},
|
||||
)
|
||||
sid = str(src["id"])
|
||||
# An active embed in flight (embedded 0 of 5) → "processing".
|
||||
IngestChunkProgressRepository(pg_conn).init_progress(sid, 5)
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.convert_source_to_wiki.delay"
|
||||
) as mock_convert, app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/convert", method="POST"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = ConvertSourceToWiki().post(sid)
|
||||
|
||||
assert response.status_code == 409
|
||||
mock_convert.assert_not_called()
|
||||
|
||||
def test_viewer_rejected_403(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import ConvertSourceToWiki
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
owner = "alice-convert"
|
||||
viewer = "bob-convert-viewer"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"files", user_id=owner, type="file",
|
||||
directory_structure={"a.md": {"type": "text/markdown"}},
|
||||
)
|
||||
sid = str(src["id"])
|
||||
_grant_team_access(pg_conn, owner, viewer, sid, "viewer")
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.convert_source_to_wiki.delay"
|
||||
) as mock_convert, app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/convert", method="POST"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": viewer}
|
||||
response = ConvertSourceToWiki().post(sid)
|
||||
|
||||
assert response.status_code == 403
|
||||
mock_convert.assert_not_called()
|
||||
|
||||
|
||||
class TestWikiPageEdit:
|
||||
def test_returns_401_unauthenticated(self, app):
|
||||
from application.api.user.sources.routes import WikiPage
|
||||
|
||||
with app.test_request_context(
|
||||
"/api/sources/x/wiki/page", method="PUT", json={}
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = None
|
||||
response = WikiPage().put("x")
|
||||
assert response.status_code == 401
|
||||
|
||||
def test_owner_writes_and_enqueues_reembed(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPage
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
from application.storage.db.repositories.wiki_pages import WikiPagesRepository
|
||||
|
||||
user = "u-edit-owner"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki", user_id=user, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.reembed_wiki_page.delay"
|
||||
) as mock_reembed, app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/page",
|
||||
method="PUT",
|
||||
json={"path": "/notes.md", "content": "body"},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = WikiPage().put(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json["page"]["path"] == "/notes.md"
|
||||
assert response.json["page"]["version"] is not None
|
||||
page = WikiPagesRepository(pg_conn).get_by_path(sid, "/notes.md")
|
||||
assert page["content"] == "body"
|
||||
mock_reembed.assert_called_once()
|
||||
assert mock_reembed.call_args.args[1] == "/notes.md"
|
||||
assert mock_reembed.call_args.kwargs["user"] == user
|
||||
|
||||
def test_team_editor_reembeds_as_owner(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPage
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
owner = "alice-edit"
|
||||
editor = "bob-edit-editor"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki", user_id=owner, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
_grant_team_access(pg_conn, owner, editor, sid, "editor")
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.reembed_wiki_page.delay"
|
||||
) as mock_reembed, app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/page",
|
||||
method="PUT",
|
||||
json={"path": "/e.md", "content": "edited"},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": editor}
|
||||
response = WikiPage().put(sid)
|
||||
|
||||
assert response.status_code == 200
|
||||
# Re-embed runs AS the owner so the owner-scoped worker load resolves.
|
||||
assert mock_reembed.call_args.kwargs["user"] == owner
|
||||
|
||||
def test_stale_version_returns_409(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPage
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
from application.storage.db.repositories.wiki_pages import WikiPagesRepository
|
||||
|
||||
user = "u-edit-conflict"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki", user_id=user, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
WikiPagesRepository(pg_conn).upsert(sid, "/c.md", "v1", updated_by=user)
|
||||
# Bump to version 2 so a stale expected_version=1 loses the race.
|
||||
WikiPagesRepository(pg_conn).upsert(sid, "/c.md", "v2", updated_by=user)
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.reembed_wiki_page.delay"
|
||||
) as mock_reembed, app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/page",
|
||||
method="PUT",
|
||||
json={"path": "/c.md", "content": "v3", "expected_version": 1},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = WikiPage().put(sid)
|
||||
|
||||
assert response.status_code == 409
|
||||
mock_reembed.assert_not_called()
|
||||
|
||||
def test_traversal_path_returns_400(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPage
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
user = "u-edit-traversal"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki", user_id=user, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.reembed_wiki_page.delay"
|
||||
), app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/page",
|
||||
method="PUT",
|
||||
json={"path": "/../secret.md", "content": "x"},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = WikiPage().put(sid)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_viewer_rejected_403(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPage
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
owner = "alice-edit-viewer"
|
||||
viewer = "bob-edit-viewer"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki", user_id=owner, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
_grant_team_access(pg_conn, owner, viewer, sid, "viewer")
|
||||
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.reembed_wiki_page.delay"
|
||||
) as mock_reembed, app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/page",
|
||||
method="PUT",
|
||||
json={"path": "/v.md", "content": "x"},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": viewer}
|
||||
response = WikiPage().put(sid)
|
||||
|
||||
assert response.status_code == 403
|
||||
mock_reembed.assert_not_called()
|
||||
@@ -86,6 +86,20 @@ class TestReingestSourceTask:
|
||||
assert result == {"status": "ok"}
|
||||
|
||||
|
||||
class TestConvertSourceToWikiTask:
|
||||
@pytest.mark.unit
|
||||
@patch("application.worker.convert_source_to_wiki_worker")
|
||||
def test_calls_convert_worker(self, mock_worker):
|
||||
from application.api.user.tasks import convert_source_to_wiki
|
||||
|
||||
mock_worker.return_value = {"status": "converted"}
|
||||
|
||||
result = convert_source_to_wiki("source123", "user1")
|
||||
|
||||
mock_worker.assert_called_once_with(ANY, "source123", "user1")
|
||||
assert result == {"status": "converted"}
|
||||
|
||||
|
||||
class TestScheduleSyncsTask:
|
||||
@pytest.mark.unit
|
||||
@patch("application.api.user.tasks.sync_worker")
|
||||
@@ -286,6 +300,8 @@ class TestDurableTaskRetryPolicy:
|
||||
"store_attachment",
|
||||
"process_agent_webhook",
|
||||
"ingest_connector_task",
|
||||
"reembed_wiki_page",
|
||||
"convert_source_to_wiki",
|
||||
],
|
||||
)
|
||||
def test_task_has_retry_config(self, task_name):
|
||||
|
||||
Reference in new issue
Block a user