From 511018914300e32456683c71258b6fa9fb93c51c Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 22 Jun 2026 18:41:54 +0100 Subject: [PATCH] feat(wiki): create-wiki route + read endpoints + minimal read-only UI Unit 4 of F-Wiki. POST /api/sources/wiki creates a type=wiki/kind=wiki source with no ingest task (optional seed page enqueues reembed). GET /wiki/pages + /wiki/page serve the tree + fresh page content, read-access gated (owner or team grant), path-validated. Frontend: 'Create Wiki' ingestor entry, read-only WikiViewer (FileTree + react-markdown, no raw HTML), sync/reingest hidden for wiki sources. --- application/api/user/sources/routes.py | 204 +++++++++++++- frontend/src/api/endpoints.ts | 4 + frontend/src/api/services/userService.ts | 13 + frontend/src/components/WikiViewer.tsx | 197 ++++++++++++++ frontend/src/locale/en.json | 10 + frontend/src/settings/Sources.tsx | 16 +- frontend/src/upload/Upload.tsx | 83 +++++- frontend/src/upload/types/ingestor.test.ts | 41 +++ frontend/src/upload/types/ingestor.ts | 25 +- tests/api/user/sources/test_wiki_routes.py | 292 +++++++++++++++++++++ 10 files changed, 874 insertions(+), 11 deletions(-) create mode 100644 frontend/src/components/WikiViewer.tsx create mode 100644 frontend/src/upload/types/ingestor.test.ts create mode 100644 tests/api/user/sources/test_wiki_routes.py diff --git a/application/api/user/sources/routes.py b/application/api/user/sources/routes.py index 3f7d447a..114464bc 100644 --- a/application/api/user/sources/routes.py +++ b/application/api/user/sources/routes.py @@ -2,14 +2,17 @@ import json import math +import uuid from flask import current_app, jsonify, make_response, redirect, request from flask_restx import fields, Namespace, Resource from pydantic import ValidationError +from application.agents.tools.path_utils import validate_tool_path from application.api import api -from application.api.user.tasks import reingest_source_task, sync_source +from application.api.user.tasks import reembed_wiki_page, reingest_source_task, sync_source from application.api.user.team_sharing import ( + can_access, effective_write_owner, visible_with_access, ) @@ -19,6 +22,11 @@ from application.storage.db.repositories.ingest_chunk_progress import ( IngestChunkProgressRepository, ) from application.storage.db.repositories.sources import SourcesRepository +from application.storage.db.repositories.wiki_pages import ( + WikiPagesRepository, + _content_hash, + rebuild_wiki_directory_structure, +) from application.storage.db.session import db_readonly, db_session from application.storage.db.source_config import SourceConfig from application.storage.storage_creator import StorageCreator @@ -26,6 +34,9 @@ from application.utils import check_required_fields from application.vectorstore.vector_creator import VectorCreator +WIKI_INDEX_PATH = "/index.md" + + sources_ns = Namespace( "sources", description="Source document management operations", path="/api" ) @@ -636,3 +647,194 @@ class SourceConfigResource(Resource): ), 200, ) + + +def _resolve_readable_source(conn, source_id, user): + """Return a source dict the caller may READ, or None. + + Read access = owner or any team grant (viewer/editor). Resolves the row + without ownership scoping only after the grant check passes. + """ + doc = SourcesRepository(conn).get_any(source_id, user) + if doc is not None: + return doc + if not can_access(conn, "source", source_id, user): + return None + return SourcesRepository(conn).get_by_id(source_id) + + +def _wiki_page_node(page): + return { + "path": page.get("path"), + "title": page.get("title"), + "token_count": page.get("token_count") or 0, + "version": page.get("version"), + "embed_status": page.get("embed_status"), + "updated_at": ( + page["updated_at"].isoformat() + if page.get("updated_at") is not None + and hasattr(page["updated_at"], "isoformat") + else page.get("updated_at") + ), + } + + +@sources_ns.route("/sources/wiki") +class CreateWikiSource(Resource): + create_wiki_model = api.model( + "CreateWikiModel", + { + "name": fields.String(required=True, description="Wiki source name"), + "initial_content": fields.String( + description="Optional markdown seed for /index.md" + ), + }, + ) + + @api.expect(create_wiki_model) + @api.doc( + description="Create an LLM-editable wiki source. No ingestion task is " + "enqueued; pages are authored via the WikiTool. An optional " + "initial_content seeds /index.md and triggers its per-page re-embed." + ) + def post(self): + decoded_token = request.decoded_token + if not decoded_token: + return make_response(jsonify({"success": False}), 401) + user = decoded_token.get("sub") + data = request.get_json(silent=True) or {} + missing_fields = check_required_fields(data, ["name"]) + if missing_fields: + return missing_fields + name = str(data["name"]).strip() + if not name: + return make_response( + jsonify({"success": False, "message": "Name is required"}), 400 + ) + initial_content = data.get("initial_content") + source_id = str(uuid.uuid4()) + try: + with db_session() as conn: + repo = SourcesRepository(conn) + repo.create( + name, + source_id=source_id, + user_id=user, + type="wiki", + config={"kind": "wiki"}, + directory_structure={}, + ) + if initial_content: + WikiPagesRepository(conn).upsert( + source_id, + WIKI_INDEX_PATH, + initial_content, + updated_by=user, + ) + rebuild_wiki_directory_structure(conn, source_id, user) + except Exception as err: + current_app.logger.error( + f"Error creating wiki source: {err}", exc_info=True + ) + return make_response(jsonify({"success": False}), 400) + if initial_content: + try: + reembed_wiki_page.delay( + source_id, + WIKI_INDEX_PATH, + _content_hash(initial_content), + user=user, + idempotency_key=( + f"reembed-wiki:{source_id}:{WIKI_INDEX_PATH}:" + f"{_content_hash(initial_content)}" + ), + ) + except Exception as err: + current_app.logger.error( + f"Error enqueuing wiki seed re-embed for {source_id}: {err}", + exc_info=True, + ) + return make_response( + jsonify({"success": True, "source_id": source_id}), 200 + ) + + +@sources_ns.route("/sources//wiki/pages") +class WikiPages(Resource): + @api.doc( + description="List a wiki source's pages (read access: owner or shared)." + ) + def get(self, source_id): + decoded_token = request.decoded_token + if not decoded_token: + return make_response(jsonify({"success": False}), 401) + user = decoded_token.get("sub") + try: + with db_readonly() as conn: + doc = _resolve_readable_source(conn, source_id, user) + if doc is None: + return make_response( + jsonify({"success": False, "message": "Source not found"}), + 404, + ) + pages = WikiPagesRepository(conn).list_for_source(str(doc["id"])) + directory_structure = doc.get("directory_structure") or {} + except Exception as err: + current_app.logger.error( + f"Error listing wiki pages for {source_id}: {err}", exc_info=True + ) + return make_response(jsonify({"success": False}), 400) + return make_response( + jsonify( + { + "success": True, + "pages": [_wiki_page_node(p) for p in pages], + "directory_structure": directory_structure, + } + ), + 200, + ) + + +@sources_ns.route("/sources//wiki/page") +class WikiPage(Resource): + @api.doc( + description="Fetch a single wiki page's content fresh from storage.", + params={"path": "Page path (e.g. /index.md)"}, + ) + def get(self, source_id): + decoded_token = request.decoded_token + if not decoded_token: + return make_response(jsonify({"success": False}), 401) + user = decoded_token.get("sub") + raw_path = request.args.get("path") + if not raw_path: + return make_response( + jsonify({"success": False, "message": "path is required"}), 400 + ) + path = validate_tool_path(raw_path) + if not path or path == "/" or path.endswith("/"): + return make_response( + jsonify({"success": False, "message": "Invalid page path"}), 400 + ) + try: + with db_readonly() as conn: + doc = _resolve_readable_source(conn, source_id, user) + if doc is None: + return make_response( + jsonify({"success": False, "message": "Source not found"}), + 404, + ) + page = WikiPagesRepository(conn).get_by_path(str(doc["id"]), path) + except Exception as err: + current_app.logger.error( + f"Error fetching wiki page for {source_id}: {err}", exc_info=True + ) + return make_response(jsonify({"success": False}), 400) + if page is None: + return make_response( + jsonify({"success": False, "message": "Page not found"}), 404 + ) + node = _wiki_page_node(page) + node["content"] = page.get("content") or "" + return make_response(jsonify({"success": True, "page": node}), 200) diff --git a/frontend/src/api/endpoints.ts b/frontend/src/api/endpoints.ts index b4a1aa90..61db92cc 100644 --- a/frontend/src/api/endpoints.ts +++ b/frontend/src/api/endpoints.ts @@ -56,6 +56,10 @@ const endpoints = { SYNC_SOURCE: '/api/sync_source', REINGEST_SOURCE: '/api/sources/reingest', SOURCE_CONFIG: (id: string) => `/api/sources/${id}/config`, + CREATE_WIKI: '/api/sources/wiki', + WIKI_PAGES: (id: string) => `/api/sources/${id}/wiki/pages`, + WIKI_PAGE: (id: string, path: string) => + `/api/sources/${id}/wiki/page?path=${encodeURIComponent(path)}`, GET_AVAILABLE_TOOLS: '/api/available_tools', GET_USER_TOOLS: '/api/get_tools', CREATE_TOOL: '/api/create_tool', diff --git a/frontend/src/api/services/userService.ts b/frontend/src/api/services/userService.ts index 6677646c..c78fb433 100644 --- a/frontend/src/api/services/userService.ts +++ b/frontend/src/api/services/userService.ts @@ -102,6 +102,19 @@ const userService = { token: string | null, ): Promise => apiClient.patch(endpoints.USER.SOURCE_CONFIG(sourceId), config, token), + createWiki: ( + data: { name: string; initial_content?: string }, + token: string | null, + ): Promise => + apiClient.post(endpoints.USER.CREATE_WIKI, data, token), + getWikiPages: (sourceId: string, token: string | null): Promise => + throttledApiClient.get(endpoints.USER.WIKI_PAGES(sourceId), token), + getWikiPage: ( + sourceId: string, + path: string, + token: string | null, + ): Promise => + throttledApiClient.get(endpoints.USER.WIKI_PAGE(sourceId, path), token), getAvailableTools: (token: string | null): Promise => apiClient.get(endpoints.USER.GET_AVAILABLE_TOOLS, token), getUserTools: (token: string | null): Promise => diff --git a/frontend/src/components/WikiViewer.tsx b/frontend/src/components/WikiViewer.tsx new file mode 100644 index 00000000..8633d847 --- /dev/null +++ b/frontend/src/components/WikiViewer.tsx @@ -0,0 +1,197 @@ +import React, { useEffect, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useSelector } from 'react-redux'; +import ReactMarkdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; + +import userService from '../api/services/userService'; +import ArrowLeft from '../assets/arrow-left.svg'; +import { selectToken } from '../preferences/preferenceSlice'; +import { Button } from './ui/button'; +import SkeletonLoader from './SkeletonLoader'; + +interface WikiPageNode { + path: string; + title?: string | null; + token_count?: number; + embed_status?: string; +} + +interface WikiViewerProps { + docId: string; + sourceName: string; + onBackToDocuments: () => void; +} + +const WikiViewer: React.FC = ({ + docId, + sourceName, + onBackToDocuments, +}) => { + const { t } = useTranslation(); + const token = useSelector(selectToken); + + const [pages, setPages] = useState([]); + const [selectedPath, setSelectedPath] = useState(null); + const [content, setContent] = useState(''); + const [loadingPages, setLoadingPages] = useState(true); + const [loadingContent, setLoadingContent] = useState(false); + + useEffect(() => { + let cancelled = false; + setLoadingPages(true); + userService + .getWikiPages(docId, token) + .then((response) => response.json()) + .then((data) => { + if (cancelled) return; + const list: WikiPageNode[] = data?.pages ?? []; + setPages(list); + if (list.length > 0) setSelectedPath(list[0].path); + }) + .catch((error) => console.error('Error loading wiki pages:', error)) + .finally(() => { + if (!cancelled) setLoadingPages(false); + }); + return () => { + cancelled = true; + }; + }, [docId, token]); + + useEffect(() => { + if (!selectedPath) { + setContent(''); + return; + } + let cancelled = false; + setLoadingContent(true); + userService + .getWikiPage(docId, selectedPath, token) + .then((response) => response.json()) + .then((data) => { + if (cancelled) return; + setContent(data?.page?.content ?? ''); + }) + .catch((error) => console.error('Error loading wiki page:', error)) + .finally(() => { + if (!cancelled) setLoadingContent(false); + }); + return () => { + cancelled = true; + }; + }, [docId, selectedPath, token]); + + return ( +
+
+ + + {sourceName} + +
+ +
+
+ {loadingPages ? ( + + ) : pages.length === 0 ? ( +

+ {t('settings.sources.wiki.empty')} +

+ ) : ( +
    + {pages.map((page) => ( +
  • + +
    • + ))} +
+ )} +
+ +
+ {loadingContent ? ( + + ) : selectedPath ? ( +
+ ( +

+ {children} +

+ ), + h2: ({ children }) => ( +

+ {children} +

+ ), + h3: ({ children }) => ( +

+ {children} +

+ ), + p: ({ children }) =>

{children}

, + ul: ({ children }) => ( +
    + {children} +
+ ), + ol: ({ children }) => ( +
    + {children} +
+ ), + a: ({ children, href }) => ( + + {children} + + ), + code: ({ children }) => ( + + {children} + + ), + }} + > + {content} + +
+ ) : ( +

+ {t('settings.sources.wiki.selectPage')} +

+ )} +
+
+
+ ); +}; + +export default WikiViewer; diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index 4f5330e7..abb40dc3 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -103,6 +103,11 @@ }, "actions": "Actions", "view": "View", + "wiki": { + "view": "View wiki", + "empty": "This wiki has no pages yet.", + "selectPage": "Select a page to view its contents." + }, "editConfig": "Source settings", "shareWithTeam": "Share with team", "deleteWarning": "Are you sure you want to delete \"{{name}}\"?", @@ -638,6 +643,7 @@ "uploadedFiles": "Uploaded Files", "cancel": "Cancel", "train": "Train", + "create": "Create", "link": "Link", "urlLink": "URL Link", "repoUrl": "Repository URL", @@ -707,6 +713,10 @@ "confluence": { "label": "Confluence", "heading": "Upload from Confluence" + }, + "wiki": { + "label": "Create Wiki", + "heading": "Create an LLM-editable wiki" } }, "connectors": { diff --git a/frontend/src/settings/Sources.tsx b/frontend/src/settings/Sources.tsx index 0206cde4..2e2572fe 100644 --- a/frontend/src/settings/Sources.tsx +++ b/frontend/src/settings/Sources.tsx @@ -45,6 +45,7 @@ import { formatDate } from '../utils/dateTimeUtils'; import FileTree from '../components/FileTree'; import ConnectorTree from '../components/ConnectorTree'; import Chunks from '../components/Chunks'; +import WikiViewer from '../components/WikiViewer'; import SourceConfigModal from './SourceConfigModal'; type SourceMenuOption = { @@ -327,10 +328,13 @@ export default function Sources({ index: number, document: Doc, ): SourceMenuOption[] => { + const isWiki = document.type === 'wiki'; const actions: SourceMenuOption[] = [ { icon: EyeView, - label: t('settings.sources.view'), + label: isWiki + ? t('settings.sources.wiki.view') + : t('settings.sources.view'), onClick: () => { setDocumentToView(document); }, @@ -380,7 +384,7 @@ export default function Sources({ }); } - if (document.id) { + if (document.id && !isWiki) { actions.push({ icon: SlidersHorizontal, label: t('settings.sources.editConfig'), @@ -428,7 +432,13 @@ export default function Sources({ return documentToView ? (
- {documentToView.isNested ? ( + {documentToView.type === 'wiki' ? ( + setDocumentToView(undefined)} + /> + ) : documentToView.isNested ? ( documentToView.type === 'connector:file' ? (
); + case 'textarea': + return ( +
+ +