Merge remote-tracking branch 'origin/main' into connectors

# Conflicts:
#	frontend/DESIGN.md
#	frontend/src/locale/de.json
#	frontend/src/locale/en.json
#	frontend/src/locale/es.json
#	frontend/src/locale/jp.json
#	frontend/src/locale/ru.json
#	frontend/src/locale/zh-TW.json
#	frontend/src/locale/zh.json
#	frontend/src/upload/Upload.tsx

docsgpt/api/user/schedules/routes.py

@@ -317,6 +317,55 @@ class AgentSchedules(Resource):
         return _ok({"schedule": _format_schedule(created)}, status=201)
 
 
+@schedules_ns.route("/agents/<string:agent_id>/schedules/stats")
+class AgentScheduleStats(Resource):
+    @api.doc(
+        description="Run stats for an agent's schedules over a recent window.",
+        params={"days": "Window in days (default 30, clamped to 1..365)"},
+    )
+    @_safe_route
+    def get(self, agent_id: str):
+        """Return run count, failures, tokens and latest failure for an agent.
+
+        Args:
+            agent_id: Agent id from the URL.
+
+        Returns:
+            A Flask response with ``days``, ``runs``, ``failed``, ``tokens``
+            and ``latest_failure`` (or an error envelope).
+        """
+        user_id = _user_id()
+        if not user_id:
+            return _err("unauthorized", 401)
+        agent = _agent_owned(agent_id, user_id)
+        if agent is None:
+            return _err("agent not found", 404)
+        try:
+            days = max(1, min(int(request.args.get("days", 30)), 365))
+        except (TypeError, ValueError):
+            days = 30
+        try:
+            with db_readonly() as conn:
+                stats = ScheduleRunsRepository(conn).stats_for_agent(
+                    str(agent["id"]), user_id, days=days,
+                )
+        except Exception as exc:
+            current_app.logger.error(
+                "schedule stats failed: %s", exc, exc_info=True,
+            )
+            return _err("internal error", 500)
+        latest = stats.get("latest_failure")
+        return _ok(
+            {
+                "days": days,
+                "runs": stats["runs"],
+                "failed": stats["failed"],
+                "tokens": stats["tokens"],
+                "latest_failure": _format_run(latest) if latest else None,
+            }
+        )
+
+
 @schedules_ns.route("/schedules/<string:schedule_id>")
 class ScheduleResource(Resource):
     @api.doc(description="Get schedule by id.")

docsgpt/api/user/sources/chunks.py

@@ -7,10 +7,11 @@ from flask_restx import fields, Namespace, Resource
 
 from docsgpt.api import api
 from docsgpt.api.user.base import get_vector_store
-from docsgpt.api.user.team_sharing import effective_write_owner
+from docsgpt.api.user.team_sharing import can_access, effective_write_owner
 from docsgpt.storage.db.repositories.sources import SourcesRepository
 from docsgpt.storage.db.session import db_readonly
 from docsgpt.utils import check_required_fields, num_tokens_from_string
+from docsgpt.vectorstore.base import InvalidChunkMetadataError
 
 sources_chunks_ns = Namespace(
     "sources", description="Source document management operations", path="/api"
@@ -18,12 +19,18 @@ sources_chunks_ns = Namespace(
 
 
 def _resolve_source(doc_id: str, user: str):
-    """Resolve a source (UUID or legacy ObjectId) for the caller.
+    """Resolve a source (UUID or legacy ObjectId) the caller may READ.
 
-    Returns the row dict (with PG UUID in ``id``) or ``None`` if missing.
+    Read access = owner or any team grant (viewer/editor). Returns the row
+    dict (with PG UUID in ``id``) or ``None`` if missing or not visible.
     """
     with db_readonly() as conn:
-        return SourcesRepository(conn).get_any(doc_id, user)
+        doc = SourcesRepository(conn).get_any(doc_id, user)
+        if doc is not None:
+            return doc
+        if not can_access(conn, "source", doc_id, user):
+            return None
+        return SourcesRepository(conn).get_by_id(doc_id)
 
 
 def _resolve_source_for_write(doc_id: str, user: str):
@@ -45,6 +52,35 @@ def _resolve_source_for_write(doc_id: str, user: str):
         return SourcesRepository(conn).get_any(doc_id, owner)
 
 
+def _remap_graph_chunk(doc: dict, old_chunk_id: str, new_chunk_id: str) -> None:
+    """Move a graphrag source's links from an edited chunk's old id to its new one.
+
+    Only needed when the store's ``update_chunk`` fell back to re-adding the
+    chunk under a new id (stores that update in place keep the id). Without
+    this the graph keeps pointing at the deleted row: the entity loses the
+    chunk and retrieval stops returning it. A failure is logged, not raised,
+    because the edit itself has already been saved.
+
+    Args:
+        doc: The resolved source row.
+        old_chunk_id: The edited chunk's previous id.
+        new_chunk_id: The id the edit was saved under.
+    """
+    from docsgpt.storage.db.source_config import SourceConfig
+
+    if SourceConfig.parse(doc.get("config")).kind != "graphrag":
+        return
+    try:
+        from docsgpt.graphrag.store import GraphStore
+
+        GraphStore().remap_chunk(str(doc["id"]), old_chunk_id, new_chunk_id)
+    except Exception as e:
+        current_app.logger.error(
+            f"Failed to remap graph links from chunk {old_chunk_id} to {new_chunk_id}: {e}",
+            exc_info=True,
+        )
+
+
 def _has_usable_token_count(metadata: dict) -> bool:
     """Whether ``metadata`` already carries a count worth showing.
 
@@ -98,6 +134,38 @@ def _with_token_counts(chunks: list) -> list:
     return chunks
 
 
+def _path_ends_with(value: str, path: str) -> bool:
+    """Return whether ``value`` is ``path`` or ends with it at a ``/`` boundary.
+
+    A bare ``endswith`` let a root ``setup.md`` also claim the chunks of
+    ``guides/setup.md`` (and ``a.md`` those of ``data.md``).
+    """
+    return bool(value) and (value == path or value.endswith(f"/{path}"))
+
+
+def _chunk_matches_path(metadata: dict, path: str) -> bool:
+    """Return whether a chunk belongs to the tree file at ``path``.
+
+    Args:
+        metadata: The chunk's stored metadata.
+        path: The file's key path in the source's ``directory_structure``.
+
+    Returns:
+        True when the chunk's ``source`` or ``file_path`` names that file, or
+        when the worker could only have keyed it by title: a remote ingest
+        (web page, Reddit post) whose chunks carry no ``file_path`` or ``key``
+        (see ``remote_worker``). Sources ingested that way stay browsable
+        without a re-ingest.
+    """
+    source = metadata.get("source") or ""
+    file_path = metadata.get("file_path") or ""
+    if _path_ends_with(source, path) or _path_ends_with(file_path, path):
+        return True
+    if "://" in source and not file_path and not metadata.get("key"):
+        return metadata.get("title") == path
+    return False
+
+
 @sources_chunks_ns.route("/get_chunks")
 class GetChunks(Resource):
     @api.doc(
@@ -141,14 +209,8 @@ class GetChunks(Resource):
             for chunk in chunks:
                 metadata = chunk.get("metadata", {})
 
-                if path:
-                    chunk_source = metadata.get("source", "")
-                    chunk_file_path = metadata.get("file_path", "")
-                    source_match = chunk_source and chunk_source.endswith(path)
-                    file_path_match = chunk_file_path and chunk_file_path.endswith(path)
-
-                    if not (source_match or file_path_match):
-                        continue
+                if path and not _chunk_matches_path(metadata, path):
+                    continue
                 if search_term:
                     text_match = search_term in chunk.get("text", "").lower()
                     title_match = search_term in metadata.get("title", "").lower()
@@ -339,13 +401,11 @@ class UpdateChunk(Resource):
             if text is not None:
                 new_metadata["token_count"] = num_tokens_from_string(new_text)
             try:
-                new_chunk_id = store.add_chunk(new_text, new_metadata)
-
-                deleted = store.delete_chunk(chunk_id)
-                if not deleted:
-                    current_app.logger.warning(
-                        f"Failed to delete old chunk {chunk_id}, but new chunk {new_chunk_id} was created"
-                    )
+                # In place where the store supports it (same id, same list
+                # position); the base fallback re-adds under a new id.
+                new_chunk_id = store.update_chunk(chunk_id, new_text, new_metadata)
+                if new_chunk_id != chunk_id:
+                    _remap_graph_chunk(doc, chunk_id, new_chunk_id)
                 return make_response(
                     jsonify(
                         {
@@ -356,8 +416,13 @@ class UpdateChunk(Resource):
                     ),
                     200,
                 )
+            except InvalidChunkMetadataError as meta_error:
+                current_app.logger.warning(
+                    f"Rejected metadata for chunk {chunk_id}: {meta_error}"
+                )
+                return make_response(jsonify({"error": "Invalid metadata"}), 400)
             except Exception as add_error:
-                current_app.logger.error(f"Failed to add updated chunk: {add_error}")
+                current_app.logger.error(f"Failed to update chunk {chunk_id}: {add_error}")
                 return make_response(
                     jsonify({"error": "Failed to update chunk - addition failed"}), 500
                 )

docsgpt/api/user/sources/routes.py

@@ -46,6 +46,9 @@ from docsgpt.vectorstore.vector_creator import VectorCreator
 
 WIKI_INDEX_PATH = "/index.md"
 
+# Longest graph-node search term forwarded to the store; longer input is truncated.
+_GRAPH_SEARCH_MAX_LEN = 200
+
 
 sources_ns = Namespace(
     "sources", description="Source document management operations", path="/api"
@@ -566,7 +569,7 @@ class DirectoryStructure(Resource):
             return make_response(jsonify({"error": "Document ID is required"}), 400)
         try:
             with db_readonly() as conn:
-                doc = SourcesRepository(conn).get_any(doc_id, user)
+                doc = _resolve_readable_source(conn, doc_id, user)
             if not doc:
                 return make_response(
                     jsonify({"error": "Document not found or access denied"}), 404
@@ -1268,18 +1271,50 @@ class EnableSourceGraphRAG(Resource):
         )
 
 
-def _graph_overview_payload(source_id, limit):
-    """Return a bounded ``{nodes, edges}`` overview for a graphrag source.
+def _graph_overview_payload(source_id: str, limit: int) -> dict:
+    """Return a bounded ``{nodes, edges, stats}`` overview for a graphrag source.
 
     An empty graph (extraction pending/capped/failed) yields empty lists rather
-    than an error, mirroring the ClassicRAG degradation guarantee.
+    than an error, mirroring the ClassicRAG degradation guarantee, and skips
+    the overview query entirely. ``stats`` carries the whole graph's totals,
+    not the bounded overview's.
+
+    Args:
+        source_id: The resolved source id.
+        limit: Requested overview size; the store clamps it.
+
+    Returns:
+        dict: ``{"nodes": [...], "edges": [...], "stats": {"nodes": int,
+        "edges": int}}``.
     """
     from docsgpt.graphrag.store import GraphStore
 
     store = GraphStore()
-    if store.count_nodes(source_id) == 0:
-        return {"nodes": [], "edges": []}
-    return store.get_graph_overview(source_id, limit)
+    node_count = store.count_nodes(source_id)
+    if node_count == 0:
+        return {"nodes": [], "edges": [], "stats": {"nodes": 0, "edges": 0}}
+    overview = store.get_graph_overview(source_id, limit)
+    return {
+        "nodes": overview["nodes"],
+        "edges": overview["edges"],
+        "stats": {"nodes": node_count, "edges": store.count_edges(source_id)},
+    }
+
+
+def _int_arg(name: str, default: int) -> int:
+    """Read an integer query arg, falling back to ``default`` when unparsable.
+
+    Args:
+        name: Query-string parameter name.
+        default: Value used when the arg is missing or not an integer.
+
+    Returns:
+        int: The parsed value or ``default``.
+    """
+    try:
+        return int(request.args.get(name, default))
+    except (TypeError, ValueError):
+        return default
 
 
 @sources_ns.route("/sources/<string:source_id>/graph")
@@ -1331,6 +1366,142 @@ class SourceGraph(Resource):
                     "success": True,
                     "nodes": overview["nodes"],
                     "edges": overview["edges"],
+                    "stats": overview["stats"],
+                }
+            ),
+            200,
+        )
+
+
+def _graph_node_page(
+    source_id: str,
+    query: str | None,
+    type_key: str | None,
+    page: int,
+    per_page: int,
+) -> tuple[dict, list]:
+    """Return ``(listing, type_facets)`` for one page of a source's graph nodes.
+
+    A graph store that is not configured, or whose tables do not exist yet
+    (no graph has ever been built on this deployment), yields an empty page,
+    matching :func:`_graph_overview_payload`. Every other failure propagates,
+    so a broken query is never shown as an empty graph.
+
+    Args:
+        source_id: The resolved source id.
+        query: Name substring filter, or ``None``.
+        type_key: Type key filter, or ``None`` for no filter.
+        page: 1-based page, already clamped.
+        per_page: Page size, already clamped.
+
+    Returns:
+        tuple: ``({"nodes": [...], "total": int}, [{"key", "label", "count"}])``.
+
+    Raises:
+        Exception: Any store failure other than a missing store or schema.
+    """
+    import psycopg
+
+    from docsgpt.graphrag.store import GraphStore
+
+    empty = ({"nodes": [], "total": 0}, [])
+    try:
+        store = GraphStore()
+    except (ValueError, ImportError) as err:
+        current_app.logger.info("Graph store unavailable, listing no nodes: %s", err)
+        return empty
+    try:
+        listing = store.list_nodes(
+            source_id,
+            query=query,
+            type_key=type_key,
+            offset=(page - 1) * per_page,
+            limit=per_page,
+        )
+        types = store.node_type_facets(source_id)
+    except psycopg.errors.UndefinedTable as err:
+        current_app.logger.info("Graph tables missing, listing no nodes: %s", err)
+        return empty
+    return listing, types
+
+
+@sources_ns.route("/sources/<string:source_id>/graph/nodes")
+class SourceGraphNodes(Resource):
+    @api.doc(
+        description="Paged, searchable list of a graphrag source's nodes, "
+        "highest degree first, plus type facets over all its nodes (read "
+        "access: owner or shared). Query params: q (name substring), type "
+        "(type key), page (1-based), per_page (1-100, default 25)."
+    )
+    def get(self, source_id: str):
+        """List one page of a source's graph nodes with type facets.
+
+        Args:
+            source_id: The source id from the URL.
+
+        Returns:
+            Response: ``{success, nodes, total, page, per_page, types}``, or
+            401/404/400 on missing auth, no read access or a failure. ``page``
+            is clamped to ``1..GRAPH_NODE_LIST_MAX_PAGE`` and ``per_page`` to
+            ``1..GRAPH_NODE_LIST_MAX_LIMIT``. A graph store that is not
+            configured or has no tables yet reads as an empty page, like the
+            overview; any other store query failure is a 400, never an empty
+            200.
+        """
+        decoded_token = request.decoded_token
+        if not decoded_token:
+            return make_response(jsonify({"success": False}), 401)
+        user = decoded_token.get("sub")
+        from docsgpt.graphrag.store import (
+            GRAPH_NODE_LIST_DEFAULT_LIMIT,
+            GRAPH_NODE_LIST_MAX_LIMIT,
+            GRAPH_NODE_LIST_MAX_PAGE,
+        )
+
+        page = max(1, min(_int_arg("page", 1), GRAPH_NODE_LIST_MAX_PAGE))
+        per_page = max(
+            1,
+            min(
+                _int_arg("per_page", GRAPH_NODE_LIST_DEFAULT_LIMIT),
+                GRAPH_NODE_LIST_MAX_LIMIT,
+            ),
+        )
+        query = (request.args.get("q") or "").strip()[:_GRAPH_SEARCH_MAX_LEN] or None
+        type_key = request.args.get("type")
+        try:
+            with db_readonly() as conn:
+                doc = _resolve_readable_source(conn, source_id, user)
+                if doc is None:
+                    return make_response(
+                        jsonify({"success": False, "message": "Source not found"}),
+                        404,
+                    )
+                resolved_source_id = str(doc["id"])
+        except Exception as err:
+            current_app.logger.error(
+                f"Error resolving source {source_id} for graph nodes: {err}",
+                exc_info=True,
+            )
+            return make_response(jsonify({"success": False}), 400)
+        try:
+            listing, types = _graph_node_page(
+                resolved_source_id, query, type_key, page, per_page
+            )
+        except Exception as err:
+            current_app.logger.error(
+                f"Error listing graph nodes for {source_id}: {err}",
+                exc_info=True,
+            )
+            return make_response(jsonify({"success": False}), 400)
+        return make_response(
+            jsonify(
+                {
+                    "success": True,
+                    "nodes": listing["nodes"],
+                    "total": listing["total"],
+                    "page": page,
+                    "per_page": per_page,
+                    "types": types,
                 }
             ),
             200,