mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 06:17:23 +00:00
refactor: rename the application package to docsgpt
The backend import package is now docsgpt, the name it will carry on PyPI; application was far too generic to install into anyone's site-packages. git mv plus a mechanical rewrite of every import, dotted string and path reference: 734 Python files, the compose files, Dockerfile, workflows, docs, setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage config, .gitignore. Behaviour is unchanged. Kept for one release: - A top-level application package whose meta-path finder resolves application.x.y to the already-imported docsgpt.x.y object, so old imports and entry points (celery -A application.app.celery, uvicorn application.asgi:asgi_app) keep working with a FutureWarning. - Celery registers every application.* task name as an alias of its docsgpt.* task on start-up, so messages queued by the previous release still run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries the previous release wrote are left unread instead of firing twice. The backend image builds from the repository root (docker build -f docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore allow-lists docsgpt/ and application/ and keeps caches, local data, .env files, the sample index files and the Dockerfile out. Compose and the image workflows point at the new context.
This commit is contained in:
1 parent
3e7f1f91b4
commit
574f96341e
984 files changed
+9847
-9682
No files matched your search
@@ -0,0 +1,57 @@
|
||||
import logging
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from docsgpt.core.url_validation import SSRFError, validate_url
|
||||
from docsgpt.parser.remote.base import BaseRemote
|
||||
from docsgpt.parser.schema.base import Document
|
||||
from docsgpt.security.safe_url import pinned_request
|
||||
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*"
|
||||
";q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Referer": "https://www.google.com/",
|
||||
"DNT": "1",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
|
||||
|
||||
class WebLoader(BaseRemote):
|
||||
def load_data(self, inputs):
|
||||
urls = inputs
|
||||
if isinstance(urls, str):
|
||||
urls = [urls]
|
||||
documents = []
|
||||
for url in urls:
|
||||
try:
|
||||
url = validate_url(url)
|
||||
except SSRFError as e:
|
||||
logging.warning(
|
||||
f"Skipping URL due to SSRF validation failure: {url} - {e}"
|
||||
)
|
||||
continue
|
||||
try:
|
||||
response = pinned_request("GET", url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
metadata = {"source": url}
|
||||
if soup.title:
|
||||
title = soup.title.get_text(strip=True)
|
||||
if title:
|
||||
metadata["title"] = title
|
||||
html_tag = soup.find("html")
|
||||
if html_tag and html_tag.get("lang"):
|
||||
metadata["language"] = html_tag.get("lang")
|
||||
documents.append(
|
||||
Document(
|
||||
soup.get_text(separator="\n", strip=True),
|
||||
extra_info=metadata,
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logging.error(f"Error processing URL {url}: {e}", exc_info=True)
|
||||
continue
|
||||
return documents
|
||||
Reference in new issue
Block a user