mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-19 22:14:31 +00:00
refactor: rename the application package to docsgpt
The backend import package is now docsgpt, the name it will carry on PyPI; application was far too generic to install into anyone's site-packages. git mv plus a mechanical rewrite of every import, dotted string and path reference: 734 Python files, the compose files, Dockerfile, workflows, docs, setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage config, .gitignore. Behaviour is unchanged. Kept for one release: - A top-level application package whose meta-path finder resolves application.x.y to the already-imported docsgpt.x.y object, so old imports and entry points (celery -A application.app.celery, uvicorn application.asgi:asgi_app) keep working with a FutureWarning. - Celery registers every application.* task name as an alias of its docsgpt.* task on start-up, so messages queued by the previous release still run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries the previous release wrote are left unread instead of firing twice. The backend image builds from the repository root (docker build -f docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore allow-lists docsgpt/ and application/ and keeps caches, local data, .env files, the sample index files and the Dockerfile out. Compose and the image workflows point at the new context.
This commit is contained in:
1 parent
3e7f1f91b4
commit
574f96341e
984 files changed
+9847
-9682
No files matched your search
@@ -0,0 +1,102 @@
|
||||
"""Shared file-extension constants for parsing and ingestion flows."""
|
||||
|
||||
import os
|
||||
|
||||
from docsgpt.parser.file.anydoc_parser import ANYDOC_GAINED_SUFFIXES
|
||||
from docsgpt.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
|
||||
|
||||
|
||||
SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS = (
|
||||
".rst",
|
||||
".md",
|
||||
".pdf",
|
||||
".txt",
|
||||
".docx",
|
||||
".csv",
|
||||
".epub",
|
||||
".html",
|
||||
".mdx",
|
||||
".json",
|
||||
".xlsx",
|
||||
".pptx",
|
||||
# Read by the HTML parsers on every engine.
|
||||
".xhtml",
|
||||
# Read by the anydoc engine (legacy/macro Office, OpenDocument, RTF).
|
||||
# Parseable regardless of DOC_PARSER_ENGINE: anydoc is a core dependency,
|
||||
# and both parser maps route these suffixes to it.
|
||||
*ANYDOC_GAINED_SUFFIXES,
|
||||
)
|
||||
|
||||
SUPPORTED_SOURCE_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg")
|
||||
|
||||
SUPPORTED_SOURCE_EXTENSIONS = (
|
||||
*SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,
|
||||
*SUPPORTED_SOURCE_IMAGE_EXTENSIONS,
|
||||
*SUPPORTED_AUDIO_EXTENSIONS,
|
||||
)
|
||||
|
||||
# Suffixes the attachment path has a dedicated parser for — exactly the keys
|
||||
# of ``get_default_file_extractor()``. Kept as a literal (importing ``bulk``
|
||||
# here would drag docling into the API process), with
|
||||
# ``tests/parser/file/test_constants.py`` asserting the two agree.
|
||||
#
|
||||
# This is *not* the whole attachment allow-list: a suffix with no parser is
|
||||
# read by ``SimpleDirectoryReader``'s plain-text fallthrough, which is right
|
||||
# for a .py or a .log and catastrophic for a video — the reader "extracts"
|
||||
# megabytes of binary garbage, truncates it, and stores it with
|
||||
# ``extraction.status == "ok"``. So unparsed suffixes are admitted on
|
||||
# content instead (``upload_limits.enforce_parseable_attachment``): text
|
||||
# passes, binary is refused. Zip is deliberately absent — source ingestion
|
||||
# extracts archives, the attachment path does not, and a zip fails the
|
||||
# content check like any other binary.
|
||||
#
|
||||
# Mirrored in ``frontend/src/constants/fileUpload.ts``; update both together.
|
||||
ATTACHMENT_PARSER_EXTENSIONS = frozenset(
|
||||
{
|
||||
*SUPPORTED_SOURCE_EXTENSIONS,
|
||||
".xhtml",
|
||||
".adoc",
|
||||
".asciidoc",
|
||||
".tiff",
|
||||
".tif",
|
||||
".bmp",
|
||||
".webp",
|
||||
".vtt",
|
||||
".xml",
|
||||
}
|
||||
# .txt has no parser of its own — it *is* the plain-text fallthrough. It
|
||||
# must be sniffed like any other unparsed suffix, or renaming a video to
|
||||
# notes.txt walks straight back into the bug this gate exists for.
|
||||
- {".txt"}
|
||||
)
|
||||
|
||||
|
||||
def attachment_extension(filename: str | None) -> str:
|
||||
"""Return the lower-cased extension of ``filename`` including the dot, or ``""``.
|
||||
|
||||
Args:
|
||||
filename: A bare filename or path; ``None`` and empty strings yield ``""``.
|
||||
|
||||
Returns:
|
||||
The last suffix in lower case (``".pdf"``), or ``""`` when there is none.
|
||||
"""
|
||||
if not filename:
|
||||
return ""
|
||||
return os.path.splitext(os.path.basename(str(filename)))[1].lower()
|
||||
|
||||
|
||||
def has_attachment_parser(filename: str | None) -> bool:
|
||||
"""Return whether an attachment's suffix has a dedicated parser.
|
||||
|
||||
A False result does not mean the file is refused: it means nothing but the
|
||||
plain-text fallthrough will read it, so it has to earn its place on
|
||||
content. The decision is by extension only, never by the mime type a
|
||||
browser reports (mobile pickers ignore ``accept`` and lie).
|
||||
|
||||
Args:
|
||||
filename: The upload's filename.
|
||||
|
||||
Returns:
|
||||
True when the suffix is in ``ATTACHMENT_PARSER_EXTENSIONS``.
|
||||
"""
|
||||
return attachment_extension(filename) in ATTACHMENT_PARSER_EXTENSIONS
|
||||
Reference in new issue
Block a user