mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 06:17:23 +00:00
feat(wiki): create-wiki route + read endpoints + minimal read-only UI
Unit 4 of F-Wiki. POST /api/sources/wiki creates a type=wiki/kind=wiki source with no ingest task (optional seed page enqueues reembed). GET /wiki/pages + /wiki/page serve the tree + fresh page content, read-access gated (owner or team grant), path-validated. Frontend: 'Create Wiki' ingestor entry, read-only WikiViewer (FileTree + react-markdown, no raw HTML), sync/reingest hidden for wiki sources.
This commit is contained in:
1 parent
98aa8db953
commit
5110189143
10 files changed
+874
-11
No files matched your search
@@ -2,14 +2,17 @@
|
||||
|
||||
import json
|
||||
import math
|
||||
import uuid
|
||||
|
||||
from flask import current_app, jsonify, make_response, redirect, request
|
||||
from flask_restx import fields, Namespace, Resource
|
||||
from pydantic import ValidationError
|
||||
|
||||
from application.agents.tools.path_utils import validate_tool_path
|
||||
from application.api import api
|
||||
from application.api.user.tasks import reingest_source_task, sync_source
|
||||
from application.api.user.tasks import reembed_wiki_page, reingest_source_task, sync_source
|
||||
from application.api.user.team_sharing import (
|
||||
can_access,
|
||||
effective_write_owner,
|
||||
visible_with_access,
|
||||
)
|
||||
@@ -19,6 +22,11 @@ from application.storage.db.repositories.ingest_chunk_progress import (
|
||||
IngestChunkProgressRepository,
|
||||
)
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
from application.storage.db.repositories.wiki_pages import (
|
||||
WikiPagesRepository,
|
||||
_content_hash,
|
||||
rebuild_wiki_directory_structure,
|
||||
)
|
||||
from application.storage.db.session import db_readonly, db_session
|
||||
from application.storage.db.source_config import SourceConfig
|
||||
from application.storage.storage_creator import StorageCreator
|
||||
@@ -26,6 +34,9 @@ from application.utils import check_required_fields
|
||||
from application.vectorstore.vector_creator import VectorCreator
|
||||
|
||||
|
||||
WIKI_INDEX_PATH = "/index.md"
|
||||
|
||||
|
||||
sources_ns = Namespace(
|
||||
"sources", description="Source document management operations", path="/api"
|
||||
)
|
||||
@@ -636,3 +647,194 @@ class SourceConfigResource(Resource):
|
||||
),
|
||||
200,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_readable_source(conn, source_id, user):
|
||||
"""Return a source dict the caller may READ, or None.
|
||||
|
||||
Read access = owner or any team grant (viewer/editor). Resolves the row
|
||||
without ownership scoping only after the grant check passes.
|
||||
"""
|
||||
doc = SourcesRepository(conn).get_any(source_id, user)
|
||||
if doc is not None:
|
||||
return doc
|
||||
if not can_access(conn, "source", source_id, user):
|
||||
return None
|
||||
return SourcesRepository(conn).get_by_id(source_id)
|
||||
|
||||
|
||||
def _wiki_page_node(page):
|
||||
return {
|
||||
"path": page.get("path"),
|
||||
"title": page.get("title"),
|
||||
"token_count": page.get("token_count") or 0,
|
||||
"version": page.get("version"),
|
||||
"embed_status": page.get("embed_status"),
|
||||
"updated_at": (
|
||||
page["updated_at"].isoformat()
|
||||
if page.get("updated_at") is not None
|
||||
and hasattr(page["updated_at"], "isoformat")
|
||||
else page.get("updated_at")
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@sources_ns.route("/sources/wiki")
|
||||
class CreateWikiSource(Resource):
|
||||
create_wiki_model = api.model(
|
||||
"CreateWikiModel",
|
||||
{
|
||||
"name": fields.String(required=True, description="Wiki source name"),
|
||||
"initial_content": fields.String(
|
||||
description="Optional markdown seed for /index.md"
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
@api.expect(create_wiki_model)
|
||||
@api.doc(
|
||||
description="Create an LLM-editable wiki source. No ingestion task is "
|
||||
"enqueued; pages are authored via the WikiTool. An optional "
|
||||
"initial_content seeds /index.md and triggers its per-page re-embed."
|
||||
)
|
||||
def post(self):
|
||||
decoded_token = request.decoded_token
|
||||
if not decoded_token:
|
||||
return make_response(jsonify({"success": False}), 401)
|
||||
user = decoded_token.get("sub")
|
||||
data = request.get_json(silent=True) or {}
|
||||
missing_fields = check_required_fields(data, ["name"])
|
||||
if missing_fields:
|
||||
return missing_fields
|
||||
name = str(data["name"]).strip()
|
||||
if not name:
|
||||
return make_response(
|
||||
jsonify({"success": False, "message": "Name is required"}), 400
|
||||
)
|
||||
initial_content = data.get("initial_content")
|
||||
source_id = str(uuid.uuid4())
|
||||
try:
|
||||
with db_session() as conn:
|
||||
repo = SourcesRepository(conn)
|
||||
repo.create(
|
||||
name,
|
||||
source_id=source_id,
|
||||
user_id=user,
|
||||
type="wiki",
|
||||
config={"kind": "wiki"},
|
||||
directory_structure={},
|
||||
)
|
||||
if initial_content:
|
||||
WikiPagesRepository(conn).upsert(
|
||||
source_id,
|
||||
WIKI_INDEX_PATH,
|
||||
initial_content,
|
||||
updated_by=user,
|
||||
)
|
||||
rebuild_wiki_directory_structure(conn, source_id, user)
|
||||
except Exception as err:
|
||||
current_app.logger.error(
|
||||
f"Error creating wiki source: {err}", exc_info=True
|
||||
)
|
||||
return make_response(jsonify({"success": False}), 400)
|
||||
if initial_content:
|
||||
try:
|
||||
reembed_wiki_page.delay(
|
||||
source_id,
|
||||
WIKI_INDEX_PATH,
|
||||
_content_hash(initial_content),
|
||||
user=user,
|
||||
idempotency_key=(
|
||||
f"reembed-wiki:{source_id}:{WIKI_INDEX_PATH}:"
|
||||
f"{_content_hash(initial_content)}"
|
||||
),
|
||||
)
|
||||
except Exception as err:
|
||||
current_app.logger.error(
|
||||
f"Error enqueuing wiki seed re-embed for {source_id}: {err}",
|
||||
exc_info=True,
|
||||
)
|
||||
return make_response(
|
||||
jsonify({"success": True, "source_id": source_id}), 200
|
||||
)
|
||||
|
||||
|
||||
@sources_ns.route("/sources/<string:source_id>/wiki/pages")
|
||||
class WikiPages(Resource):
|
||||
@api.doc(
|
||||
description="List a wiki source's pages (read access: owner or shared)."
|
||||
)
|
||||
def get(self, source_id):
|
||||
decoded_token = request.decoded_token
|
||||
if not decoded_token:
|
||||
return make_response(jsonify({"success": False}), 401)
|
||||
user = decoded_token.get("sub")
|
||||
try:
|
||||
with db_readonly() as conn:
|
||||
doc = _resolve_readable_source(conn, source_id, user)
|
||||
if doc is None:
|
||||
return make_response(
|
||||
jsonify({"success": False, "message": "Source not found"}),
|
||||
404,
|
||||
)
|
||||
pages = WikiPagesRepository(conn).list_for_source(str(doc["id"]))
|
||||
directory_structure = doc.get("directory_structure") or {}
|
||||
except Exception as err:
|
||||
current_app.logger.error(
|
||||
f"Error listing wiki pages for {source_id}: {err}", exc_info=True
|
||||
)
|
||||
return make_response(jsonify({"success": False}), 400)
|
||||
return make_response(
|
||||
jsonify(
|
||||
{
|
||||
"success": True,
|
||||
"pages": [_wiki_page_node(p) for p in pages],
|
||||
"directory_structure": directory_structure,
|
||||
}
|
||||
),
|
||||
200,
|
||||
)
|
||||
|
||||
|
||||
@sources_ns.route("/sources/<string:source_id>/wiki/page")
|
||||
class WikiPage(Resource):
|
||||
@api.doc(
|
||||
description="Fetch a single wiki page's content fresh from storage.",
|
||||
params={"path": "Page path (e.g. /index.md)"},
|
||||
)
|
||||
def get(self, source_id):
|
||||
decoded_token = request.decoded_token
|
||||
if not decoded_token:
|
||||
return make_response(jsonify({"success": False}), 401)
|
||||
user = decoded_token.get("sub")
|
||||
raw_path = request.args.get("path")
|
||||
if not raw_path:
|
||||
return make_response(
|
||||
jsonify({"success": False, "message": "path is required"}), 400
|
||||
)
|
||||
path = validate_tool_path(raw_path)
|
||||
if not path or path == "/" or path.endswith("/"):
|
||||
return make_response(
|
||||
jsonify({"success": False, "message": "Invalid page path"}), 400
|
||||
)
|
||||
try:
|
||||
with db_readonly() as conn:
|
||||
doc = _resolve_readable_source(conn, source_id, user)
|
||||
if doc is None:
|
||||
return make_response(
|
||||
jsonify({"success": False, "message": "Source not found"}),
|
||||
404,
|
||||
)
|
||||
page = WikiPagesRepository(conn).get_by_path(str(doc["id"]), path)
|
||||
except Exception as err:
|
||||
current_app.logger.error(
|
||||
f"Error fetching wiki page for {source_id}: {err}", exc_info=True
|
||||
)
|
||||
return make_response(jsonify({"success": False}), 400)
|
||||
if page is None:
|
||||
return make_response(
|
||||
jsonify({"success": False, "message": "Page not found"}), 404
|
||||
)
|
||||
node = _wiki_page_node(page)
|
||||
node["content"] = page.get("content") or ""
|
||||
return make_response(jsonify({"success": True, "page": node}), 200)
|
||||
@@ -56,6 +56,10 @@ const endpoints = {
|
||||
SYNC_SOURCE: '/api/sync_source',
|
||||
REINGEST_SOURCE: '/api/sources/reingest',
|
||||
SOURCE_CONFIG: (id: string) => `/api/sources/${id}/config`,
|
||||
CREATE_WIKI: '/api/sources/wiki',
|
||||
WIKI_PAGES: (id: string) => `/api/sources/${id}/wiki/pages`,
|
||||
WIKI_PAGE: (id: string, path: string) =>
|
||||
`/api/sources/${id}/wiki/page?path=${encodeURIComponent(path)}`,
|
||||
GET_AVAILABLE_TOOLS: '/api/available_tools',
|
||||
GET_USER_TOOLS: '/api/get_tools',
|
||||
CREATE_TOOL: '/api/create_tool',
|
||||
|
||||
@@ -102,6 +102,19 @@ const userService = {
|
||||
token: string | null,
|
||||
): Promise<Response> =>
|
||||
apiClient.patch(endpoints.USER.SOURCE_CONFIG(sourceId), config, token),
|
||||
createWiki: (
|
||||
data: { name: string; initial_content?: string },
|
||||
token: string | null,
|
||||
): Promise<Response> =>
|
||||
apiClient.post(endpoints.USER.CREATE_WIKI, data, token),
|
||||
getWikiPages: (sourceId: string, token: string | null): Promise<Response> =>
|
||||
throttledApiClient.get(endpoints.USER.WIKI_PAGES(sourceId), token),
|
||||
getWikiPage: (
|
||||
sourceId: string,
|
||||
path: string,
|
||||
token: string | null,
|
||||
): Promise<Response> =>
|
||||
throttledApiClient.get(endpoints.USER.WIKI_PAGE(sourceId, path), token),
|
||||
getAvailableTools: (token: string | null): Promise<any> =>
|
||||
apiClient.get(endpoints.USER.GET_AVAILABLE_TOOLS, token),
|
||||
getUserTools: (token: string | null): Promise<any> =>
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useSelector } from 'react-redux';
|
||||
import ReactMarkdown from 'react-markdown';
|
||||
import remarkGfm from 'remark-gfm';
|
||||
|
||||
import userService from '../api/services/userService';
|
||||
import ArrowLeft from '../assets/arrow-left.svg';
|
||||
import { selectToken } from '../preferences/preferenceSlice';
|
||||
import { Button } from './ui/button';
|
||||
import SkeletonLoader from './SkeletonLoader';
|
||||
|
||||
interface WikiPageNode {
|
||||
path: string;
|
||||
title?: string | null;
|
||||
token_count?: number;
|
||||
embed_status?: string;
|
||||
}
|
||||
|
||||
interface WikiViewerProps {
|
||||
docId: string;
|
||||
sourceName: string;
|
||||
onBackToDocuments: () => void;
|
||||
}
|
||||
|
||||
const WikiViewer: React.FC<WikiViewerProps> = ({
|
||||
docId,
|
||||
sourceName,
|
||||
onBackToDocuments,
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const token = useSelector(selectToken);
|
||||
|
||||
const [pages, setPages] = useState<WikiPageNode[]>([]);
|
||||
const [selectedPath, setSelectedPath] = useState<string | null>(null);
|
||||
const [content, setContent] = useState<string>('');
|
||||
const [loadingPages, setLoadingPages] = useState(true);
|
||||
const [loadingContent, setLoadingContent] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
setLoadingPages(true);
|
||||
userService
|
||||
.getWikiPages(docId, token)
|
||||
.then((response) => response.json())
|
||||
.then((data) => {
|
||||
if (cancelled) return;
|
||||
const list: WikiPageNode[] = data?.pages ?? [];
|
||||
setPages(list);
|
||||
if (list.length > 0) setSelectedPath(list[0].path);
|
||||
})
|
||||
.catch((error) => console.error('Error loading wiki pages:', error))
|
||||
.finally(() => {
|
||||
if (!cancelled) setLoadingPages(false);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [docId, token]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedPath) {
|
||||
setContent('');
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
setLoadingContent(true);
|
||||
userService
|
||||
.getWikiPage(docId, selectedPath, token)
|
||||
.then((response) => response.json())
|
||||
.then((data) => {
|
||||
if (cancelled) return;
|
||||
setContent(data?.page?.content ?? '');
|
||||
})
|
||||
.catch((error) => console.error('Error loading wiki page:', error))
|
||||
.finally(() => {
|
||||
if (!cancelled) setLoadingContent(false);
|
||||
});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [docId, selectedPath, token]);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col">
|
||||
<div className="mb-4 flex items-center">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="icon-sm"
|
||||
className="text-muted-foreground mr-3 h-[29px] w-[29px] rounded-full p-2 dark:border-0"
|
||||
onClick={onBackToDocuments}
|
||||
aria-label={t('settings.sources.backToAll')}
|
||||
>
|
||||
<img src={ArrowLeft} alt="left-arrow" className="h-3 w-3" />
|
||||
</Button>
|
||||
<span className="text-primary font-semibold wrap-break-word">
|
||||
{sourceName}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-4 md:flex-row">
|
||||
<div className="border-border md:w-64 md:shrink-0 md:border-r md:pr-4">
|
||||
{loadingPages ? (
|
||||
<SkeletonLoader count={3} />
|
||||
) : pages.length === 0 ? (
|
||||
<p className="text-muted-foreground py-2 text-sm">
|
||||
{t('settings.sources.wiki.empty')}
|
||||
</p>
|
||||
) : (
|
||||
<ul className="flex flex-col gap-1">
|
||||
{pages.map((page) => (
|
||||
<li key={page.path}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setSelectedPath(page.path)}
|
||||
className={`w-full truncate rounded-lg px-3 py-2 text-left text-sm transition-colors ${
|
||||
selectedPath === page.path
|
||||
? 'bg-muted text-foreground dark:bg-accent'
|
||||
: 'text-muted-foreground hover:bg-muted dark:hover:bg-accent'
|
||||
}`}
|
||||
title={page.path}
|
||||
>
|
||||
{page.title || page.path}
|
||||
</button>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="min-w-0 flex-1">
|
||||
{loadingContent ? (
|
||||
<SkeletonLoader count={4} />
|
||||
) : selectedPath ? (
|
||||
<article className="text-foreground max-w-none text-sm leading-relaxed wrap-break-word">
|
||||
<ReactMarkdown
|
||||
remarkPlugins={[remarkGfm]}
|
||||
components={{
|
||||
h1: ({ children }) => (
|
||||
<h1 className="mt-4 mb-2 text-xl font-semibold">
|
||||
{children}
|
||||
</h1>
|
||||
),
|
||||
h2: ({ children }) => (
|
||||
<h2 className="mt-4 mb-2 text-lg font-semibold">
|
||||
{children}
|
||||
</h2>
|
||||
),
|
||||
h3: ({ children }) => (
|
||||
<h3 className="mt-3 mb-2 text-base font-semibold">
|
||||
{children}
|
||||
</h3>
|
||||
),
|
||||
p: ({ children }) => <p className="mb-3">{children}</p>,
|
||||
ul: ({ children }) => (
|
||||
<ul className="mb-3 list-inside list-disc pl-4">
|
||||
{children}
|
||||
</ul>
|
||||
),
|
||||
ol: ({ children }) => (
|
||||
<ol className="mb-3 list-inside list-decimal pl-4">
|
||||
{children}
|
||||
</ol>
|
||||
),
|
||||
a: ({ children, href }) => (
|
||||
<a
|
||||
href={href}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="text-primary underline"
|
||||
>
|
||||
{children}
|
||||
</a>
|
||||
),
|
||||
code: ({ children }) => (
|
||||
<code className="dark:bg-accent rounded-md bg-gray-200 px-1.5 py-0.5 text-xs">
|
||||
{children}
|
||||
</code>
|
||||
),
|
||||
}}
|
||||
>
|
||||
{content}
|
||||
</ReactMarkdown>
|
||||
</article>
|
||||
) : (
|
||||
<p className="text-muted-foreground py-2 text-sm">
|
||||
{t('settings.sources.wiki.selectPage')}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default WikiViewer;
|
||||
@@ -103,6 +103,11 @@
|
||||
},
|
||||
"actions": "Actions",
|
||||
"view": "View",
|
||||
"wiki": {
|
||||
"view": "View wiki",
|
||||
"empty": "This wiki has no pages yet.",
|
||||
"selectPage": "Select a page to view its contents."
|
||||
},
|
||||
"editConfig": "Source settings",
|
||||
"shareWithTeam": "Share with team",
|
||||
"deleteWarning": "Are you sure you want to delete \"{{name}}\"?",
|
||||
@@ -638,6 +643,7 @@
|
||||
"uploadedFiles": "Uploaded Files",
|
||||
"cancel": "Cancel",
|
||||
"train": "Train",
|
||||
"create": "Create",
|
||||
"link": "Link",
|
||||
"urlLink": "URL Link",
|
||||
"repoUrl": "Repository URL",
|
||||
@@ -707,6 +713,10 @@
|
||||
"confluence": {
|
||||
"label": "Confluence",
|
||||
"heading": "Upload from Confluence"
|
||||
},
|
||||
"wiki": {
|
||||
"label": "Create Wiki",
|
||||
"heading": "Create an LLM-editable wiki"
|
||||
}
|
||||
},
|
||||
"connectors": {
|
||||
|
||||
@@ -45,6 +45,7 @@ import { formatDate } from '../utils/dateTimeUtils';
|
||||
import FileTree from '../components/FileTree';
|
||||
import ConnectorTree from '../components/ConnectorTree';
|
||||
import Chunks from '../components/Chunks';
|
||||
import WikiViewer from '../components/WikiViewer';
|
||||
import SourceConfigModal from './SourceConfigModal';
|
||||
|
||||
type SourceMenuOption = {
|
||||
@@ -327,10 +328,13 @@ export default function Sources({
|
||||
index: number,
|
||||
document: Doc,
|
||||
): SourceMenuOption[] => {
|
||||
const isWiki = document.type === 'wiki';
|
||||
const actions: SourceMenuOption[] = [
|
||||
{
|
||||
icon: EyeView,
|
||||
label: t('settings.sources.view'),
|
||||
label: isWiki
|
||||
? t('settings.sources.wiki.view')
|
||||
: t('settings.sources.view'),
|
||||
onClick: () => {
|
||||
setDocumentToView(document);
|
||||
},
|
||||
@@ -380,7 +384,7 @@ export default function Sources({
|
||||
});
|
||||
}
|
||||
|
||||
if (document.id) {
|
||||
if (document.id && !isWiki) {
|
||||
actions.push({
|
||||
icon: SlidersHorizontal,
|
||||
label: t('settings.sources.editConfig'),
|
||||
@@ -428,7 +432,13 @@ export default function Sources({
|
||||
|
||||
return documentToView ? (
|
||||
<div className="mt-8 flex flex-col">
|
||||
{documentToView.isNested ? (
|
||||
{documentToView.type === 'wiki' ? (
|
||||
<WikiViewer
|
||||
docId={documentToView.id || ''}
|
||||
sourceName={documentToView.name}
|
||||
onBackToDocuments={() => setDocumentToView(undefined)}
|
||||
/>
|
||||
) : documentToView.isNested ? (
|
||||
documentToView.type === 'connector:file' ? (
|
||||
<ConnectorTree
|
||||
docId={documentToView.id || ''}
|
||||
|
||||
@@ -5,6 +5,7 @@ import { useTranslation } from 'react-i18next';
|
||||
import { useDispatch, useSelector, useStore } from 'react-redux';
|
||||
|
||||
import type { RootState } from '../store';
|
||||
import userService from '../api/services/userService';
|
||||
import { getSessionToken } from '../utils/providerUtils';
|
||||
import { Button } from '../components/ui/button';
|
||||
import { Input } from '../components/ui/input';
|
||||
@@ -210,6 +211,34 @@ function Upload({
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
case 'textarea':
|
||||
return (
|
||||
<div key={field.name} className="flex flex-col gap-2">
|
||||
<Label
|
||||
htmlFor={`field-${field.name}`}
|
||||
className="text-foreground text-sm"
|
||||
>
|
||||
{field.label}
|
||||
</Label>
|
||||
<textarea
|
||||
id={`field-${field.name}`}
|
||||
name={field.name}
|
||||
value={String(
|
||||
ingestor.config[field.name as keyof typeof ingestor.config] ??
|
||||
'',
|
||||
)}
|
||||
onChange={(e) =>
|
||||
handleIngestorChange(
|
||||
field.name as keyof IngestorConfig['config'],
|
||||
e.target.value,
|
||||
)
|
||||
}
|
||||
required={isRequired}
|
||||
rows={8}
|
||||
className="border-border bg-card text-foreground focus:border-primary w-full resize-y rounded-2xl border p-3 text-sm outline-none"
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
case 'local_file_picker':
|
||||
return (
|
||||
<div key={field.name}>
|
||||
@@ -712,6 +741,42 @@ function Upload({
|
||||
xhr.send(formData);
|
||||
};
|
||||
|
||||
const createWiki = (clientTaskId: string) => {
|
||||
dispatch(
|
||||
updateUploadTask({
|
||||
id: clientTaskId,
|
||||
updates: { status: 'training', progress: 0 },
|
||||
}),
|
||||
);
|
||||
userService
|
||||
.createWiki(
|
||||
{
|
||||
name: ingestor.name,
|
||||
initial_content: String(ingestor.config.initial_content ?? ''),
|
||||
},
|
||||
token,
|
||||
)
|
||||
.then((response) => response.json())
|
||||
.then((data) => {
|
||||
if (!data?.success) {
|
||||
handleTaskFailure(clientTaskId, data?.message);
|
||||
return;
|
||||
}
|
||||
dispatch(
|
||||
updateUploadTask({
|
||||
id: clientTaskId,
|
||||
updates: {
|
||||
sourceId: data.source_id,
|
||||
status: 'completed',
|
||||
progress: 100,
|
||||
},
|
||||
}),
|
||||
);
|
||||
onSuccessfulUpload?.();
|
||||
})
|
||||
.catch(() => handleTaskFailure(clientTaskId));
|
||||
};
|
||||
|
||||
const handleClose = useCallback(() => {
|
||||
resetUploaderState();
|
||||
setModalState('INACTIVE');
|
||||
@@ -745,7 +810,9 @@ function Upload({
|
||||
}),
|
||||
);
|
||||
|
||||
if (hasLocalFilePicker) {
|
||||
if (ingestor.type === 'wiki') {
|
||||
createWiki(clientTaskId);
|
||||
} else if (hasLocalFilePicker) {
|
||||
uploadFile(clientTaskId);
|
||||
} else {
|
||||
uploadRemote(clientTaskId);
|
||||
@@ -968,10 +1035,12 @@ function Upload({
|
||||
className="w-full"
|
||||
/>
|
||||
{renderFormFields()}
|
||||
<RetrievalOptions
|
||||
value={retrievalOptions}
|
||||
onChange={setRetrievalOptions}
|
||||
/>
|
||||
{ingestor.type !== 'wiki' && (
|
||||
<RetrievalOptions
|
||||
value={retrievalOptions}
|
||||
onChange={setRetrievalOptions}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -1004,7 +1073,9 @@ function Upload({
|
||||
: ''
|
||||
}`}
|
||||
>
|
||||
{t('modals.uploadDoc.train')}
|
||||
{ingestor.type === 'wiki'
|
||||
? t('modals.uploadDoc.create')
|
||||
: t('modals.uploadDoc.train')}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
IngestorDefaultConfigs,
|
||||
IngestorFormSchemas,
|
||||
getIngestorSchema,
|
||||
} from './ingestor';
|
||||
|
||||
describe('wiki ingestor schema', () => {
|
||||
it('registers a "wiki" ingestor with the Create Wiki label', () => {
|
||||
const schema = getIngestorSchema('wiki');
|
||||
expect(schema).toBeDefined();
|
||||
expect(schema?.label).toBe('Create Wiki');
|
||||
});
|
||||
|
||||
it('exposes an optional initial_content textarea field', () => {
|
||||
const schema = getIngestorSchema('wiki');
|
||||
const field = schema?.fields.find((f) => f.name === 'initial_content');
|
||||
expect(field).toBeDefined();
|
||||
expect(field?.type).toBe('textarea');
|
||||
expect(field?.required ?? false).toBe(false);
|
||||
});
|
||||
|
||||
it('does not require a remote file picker (create-only, no ingest)', () => {
|
||||
const schema = getIngestorSchema('wiki');
|
||||
const pickerTypes = (schema?.fields ?? []).map((f) => f.type);
|
||||
expect(pickerTypes).not.toContain('local_file_picker');
|
||||
expect(pickerTypes).not.toContain('remote_file_picker');
|
||||
});
|
||||
|
||||
it('provides a default config for the wiki type', () => {
|
||||
expect(IngestorDefaultConfigs.wiki).toEqual({
|
||||
name: '',
|
||||
config: { initial_content: '' },
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps the wiki schema discoverable in the form schema list', () => {
|
||||
expect(IngestorFormSchemas.some((s) => s.key === 'wiki')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@ import DriveIcon from '../../assets/drive.svg';
|
||||
import S3Icon from '../../assets/s3.svg';
|
||||
import SharePoint from '../../assets/sharepoint.svg';
|
||||
import ConfluenceIcon from '../../assets/confluence.svg';
|
||||
import BookIcon from '../../assets/book.svg';
|
||||
|
||||
export type IngestorType =
|
||||
| 'confluence'
|
||||
@@ -17,7 +18,8 @@ export type IngestorType =
|
||||
| 'google_drive'
|
||||
| 'local_file'
|
||||
| 's3'
|
||||
| 'share_point';
|
||||
| 'share_point'
|
||||
| 'wiki';
|
||||
|
||||
export interface IngestorConfig {
|
||||
type: IngestorType | null;
|
||||
@@ -37,6 +39,7 @@ export type FieldType =
|
||||
| 'number'
|
||||
| 'enum'
|
||||
| 'boolean'
|
||||
| 'textarea'
|
||||
| 'local_file_picker'
|
||||
| 'remote_file_picker'
|
||||
| 'google_drive_picker'
|
||||
@@ -235,6 +238,20 @@ export const IngestorFormSchemas: IngestorSchema[] = [
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
key: 'wiki',
|
||||
label: 'Create Wiki',
|
||||
icon: BookIcon,
|
||||
heading: 'Create an LLM-editable wiki',
|
||||
fields: [
|
||||
{
|
||||
name: 'initial_content',
|
||||
label: 'Initial content (optional)',
|
||||
type: 'textarea',
|
||||
required: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
export const IngestorDefaultConfigs: Record<
|
||||
@@ -289,6 +306,12 @@ export const IngestorDefaultConfigs: Record<
|
||||
folder_ids: '',
|
||||
},
|
||||
},
|
||||
wiki: {
|
||||
name: '',
|
||||
config: {
|
||||
initial_content: '',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export interface IngestorOption {
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
"""Tests for the wiki source routes in application/api/user/sources/routes.py."""
|
||||
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from flask import Flask
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
return Flask(__name__)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_db(conn):
|
||||
@contextmanager
|
||||
def _yield():
|
||||
yield conn
|
||||
|
||||
with patch(
|
||||
"application.api.user.sources.routes.db_session", _yield
|
||||
), patch(
|
||||
"application.api.user.sources.routes.db_readonly", _yield
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def _grant_team_access(pg_conn, owner, member, source_id, access_level):
|
||||
from application.storage.db.repositories.team_members import (
|
||||
TeamMembersRepository,
|
||||
)
|
||||
from application.storage.db.repositories.team_resource_grants import (
|
||||
TeamResourceGrantsRepository,
|
||||
)
|
||||
from application.storage.db.repositories.teams import TeamsRepository
|
||||
|
||||
team = TeamsRepository(pg_conn).create(
|
||||
"Acme", f"acme-{uuid.uuid4().hex[:8]}", owner
|
||||
)
|
||||
TeamMembersRepository(pg_conn).add_member(
|
||||
team["id"], member, role="team_member"
|
||||
)
|
||||
TeamResourceGrantsRepository(pg_conn).grant(
|
||||
team["id"], "source", source_id, owner_id=owner, granted_by=owner,
|
||||
access_level=access_level,
|
||||
)
|
||||
|
||||
|
||||
class TestCreateWikiSource:
|
||||
def test_returns_401_unauthenticated(self, app):
|
||||
from application.api.user.sources.routes import CreateWikiSource
|
||||
|
||||
with app.test_request_context(
|
||||
"/api/sources/wiki", method="POST", json={"name": "w"}
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = None
|
||||
response = CreateWikiSource().post()
|
||||
assert response.status_code == 401
|
||||
|
||||
def test_returns_400_missing_name(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import CreateWikiSource
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
"/api/sources/wiki", method="POST", json={}
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": "u-wiki"}
|
||||
response = CreateWikiSource().post()
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_creates_row_without_ingest(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import CreateWikiSource
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
user = "u-wiki-create"
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.reembed_wiki_page.delay"
|
||||
) as mock_reembed, patch(
|
||||
"application.api.user.tasks.ingest.delay"
|
||||
) as mock_ingest, patch(
|
||||
"application.api.user.tasks.reingest_source_task.delay"
|
||||
) as mock_reingest, app.test_request_context(
|
||||
"/api/sources/wiki", method="POST", json={"name": "My Wiki"}
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = CreateWikiSource().post()
|
||||
|
||||
assert response.status_code == 200
|
||||
source_id = response.json["source_id"]
|
||||
row = SourcesRepository(pg_conn).get_any(source_id, user)
|
||||
assert row["type"] == "wiki"
|
||||
assert row["config"]["kind"] == "wiki"
|
||||
# No seed content → no re-embed, and never any ingest/reingest task.
|
||||
mock_reembed.assert_not_called()
|
||||
mock_ingest.assert_not_called()
|
||||
mock_reingest.assert_not_called()
|
||||
|
||||
def test_seed_page_roundtrips_and_only_seed_reembeds(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import (
|
||||
CreateWikiSource,
|
||||
WikiPage,
|
||||
WIKI_INDEX_PATH,
|
||||
)
|
||||
|
||||
user = "u-wiki-seed"
|
||||
with _patch_db(pg_conn), patch(
|
||||
"application.api.user.sources.routes.reembed_wiki_page.delay"
|
||||
) as mock_reembed, patch(
|
||||
"application.api.user.tasks.ingest.delay"
|
||||
) as mock_ingest, patch(
|
||||
"application.api.user.tasks.reingest_source_task.delay"
|
||||
) as mock_reingest, app.test_request_context(
|
||||
"/api/sources/wiki",
|
||||
method="POST",
|
||||
json={"name": "Seeded", "initial_content": "# Hello\nworld"},
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = CreateWikiSource().post()
|
||||
|
||||
assert response.status_code == 200
|
||||
source_id = response.json["source_id"]
|
||||
# Re-embed fires once, only for the seed page; no ingest/reingest ever.
|
||||
mock_reembed.assert_called_once()
|
||||
assert mock_reembed.call_args.args[0] == source_id
|
||||
assert mock_reembed.call_args.args[1] == WIKI_INDEX_PATH
|
||||
assert mock_reembed.call_args.kwargs["user"] == user
|
||||
mock_ingest.assert_not_called()
|
||||
mock_reingest.assert_not_called()
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{source_id}/wiki/page?path={WIKI_INDEX_PATH}"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
page_response = WikiPage().get(source_id)
|
||||
assert page_response.status_code == 200
|
||||
assert page_response.json["page"]["content"] == "# Hello\nworld"
|
||||
|
||||
|
||||
class TestWikiPages:
|
||||
def test_returns_401_unauthenticated(self, app):
|
||||
from application.api.user.sources.routes import WikiPages
|
||||
|
||||
with app.test_request_context("/api/sources/x/wiki/pages"):
|
||||
from flask import request
|
||||
request.decoded_token = None
|
||||
response = WikiPages().get("x")
|
||||
assert response.status_code == 401
|
||||
|
||||
def test_owner_lists_pages(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPages
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
from application.storage.db.repositories.wiki_pages import WikiPagesRepository
|
||||
|
||||
user = "u-wiki-list"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki-list", user_id=user, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
WikiPagesRepository(pg_conn).upsert(sid, "/index.md", "root", updated_by=user)
|
||||
WikiPagesRepository(pg_conn).upsert(sid, "/docs/a.md", "a", updated_by=user)
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/pages"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = WikiPages().get(sid)
|
||||
assert response.status_code == 200
|
||||
paths = {p["path"] for p in response.json["pages"]}
|
||||
assert paths == {"/index.md", "/docs/a.md"}
|
||||
|
||||
def test_non_owner_without_grant_404(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPages
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
owner = "u-wiki-owner"
|
||||
stranger = "u-wiki-stranger"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki-private", user_id=owner, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/pages"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": stranger}
|
||||
response = WikiPages().get(sid)
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_team_viewer_can_read(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPages
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
from application.storage.db.repositories.wiki_pages import WikiPagesRepository
|
||||
|
||||
owner = "alice-wiki"
|
||||
viewer = "bob-wiki-viewer"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki-shared", user_id=owner, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
WikiPagesRepository(pg_conn).upsert(sid, "/index.md", "x", updated_by=owner)
|
||||
_grant_team_access(pg_conn, owner, viewer, sid, "viewer")
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/pages"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": viewer}
|
||||
response = WikiPages().get(sid)
|
||||
assert response.status_code == 200
|
||||
assert {p["path"] for p in response.json["pages"]} == {"/index.md"}
|
||||
|
||||
|
||||
class TestWikiPage:
|
||||
def test_returns_400_missing_path(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPage
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
user = "u-wiki-page-nopath"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki", user_id=user, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/page"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = WikiPage().get(sid)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_returns_400_traversal_path(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPage
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
user = "u-wiki-page-traversal"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki", user_id=user, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/page?path=/../secret.md"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = WikiPage().get(sid)
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_returns_404_unknown_page(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPage
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
|
||||
user = "u-wiki-page-missing"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki", user_id=user, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/page?path=/nope.md"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": user}
|
||||
response = WikiPage().get(sid)
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_non_owner_without_grant_404(self, app, pg_conn):
|
||||
from application.api.user.sources.routes import WikiPage
|
||||
from application.storage.db.repositories.sources import SourcesRepository
|
||||
from application.storage.db.repositories.wiki_pages import WikiPagesRepository
|
||||
|
||||
owner = "u-wiki-pg-owner"
|
||||
stranger = "u-wiki-pg-stranger"
|
||||
src = SourcesRepository(pg_conn).create(
|
||||
"wiki", user_id=owner, type="wiki", config={"kind": "wiki"}
|
||||
)
|
||||
sid = str(src["id"])
|
||||
WikiPagesRepository(pg_conn).upsert(sid, "/index.md", "secret", updated_by=owner)
|
||||
|
||||
with _patch_db(pg_conn), app.test_request_context(
|
||||
f"/api/sources/{sid}/wiki/page?path=/index.md"
|
||||
):
|
||||
from flask import request
|
||||
request.decoded_token = {"sub": stranger}
|
||||
response = WikiPage().get(sid)
|
||||
assert response.status_code == 404
|
||||
Reference in new issue
Block a user