mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-14 16:16:48 +00:00
refactor: rename the application package to docsgpt
The backend import package is now docsgpt, the name it will carry on PyPI; application was far too generic to install into anyone's site-packages. git mv plus a mechanical rewrite of every import, dotted string and path reference: 734 Python files, the compose files, Dockerfile, workflows, docs, setup scripts, devcontainer, k8s manifests, vscode config, pytest and coverage config, .gitignore. Behaviour is unchanged. Kept for one release: - A top-level application package whose meta-path finder resolves application.x.y to the already-imported docsgpt.x.y object, so old imports and entry points (celery -A application.app.celery, uvicorn application.asgi:asgi_app) keep working with a FutureWarning. - Celery registers every application.* task name as an alias of its docsgpt.* task on start-up, so messages queued by the previous release still run. The redbeat key prefix moves to redbeat:docsgpt:v2: so schedule entries the previous release wrote are left unread instead of firing twice. The backend image builds from the repository root (docker build -f docsgpt/Dockerfile .) so it can ship the alias package; a root .dockerignore allow-lists docsgpt/ and application/ and keeps caches, local data, .env files, the sample index files and the Dockerfile out. Compose and the image workflows point at the new context.
This commit is contained in:
1 parent
3e7f1f91b4
commit
574f96341e
984 files changed
+9847
-9682
No files matched your search
@@ -0,0 +1,89 @@
|
||||
import json
|
||||
from typing import Any, Iterable, List
|
||||
|
||||
from docsgpt.parser.remote.base import BaseRemote
|
||||
from docsgpt.parser.schema.base import Document
|
||||
|
||||
REQUIRED_FIELDS = ["client_id", "client_secret", "user_agent", "search_queries"]
|
||||
|
||||
|
||||
class RedditPostsLoaderRemote(BaseRemote):
|
||||
"""Load Reddit posts through praw.
|
||||
|
||||
Create an app at https://www.reddit.com/prefs/apps/ to obtain the
|
||||
``client_id`` and ``client_secret``.
|
||||
"""
|
||||
|
||||
def load_data(self, inputs) -> List[Document]:
|
||||
"""Load posts for every configured subreddit or user.
|
||||
|
||||
Args:
|
||||
inputs: JSON string holding the praw credentials plus
|
||||
``search_queries``, and optionally ``mode``
|
||||
(``subreddit``/``username``), ``categories`` and
|
||||
``number_posts``.
|
||||
|
||||
Returns:
|
||||
One document per post, carrying the post body as text.
|
||||
"""
|
||||
try:
|
||||
data = json.loads(inputs)
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"Invalid JSON input: {e}")
|
||||
|
||||
missing_fields = [field for field in REQUIRED_FIELDS if field not in data]
|
||||
if missing_fields:
|
||||
raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
|
||||
|
||||
mode = data.get("mode", "subreddit")
|
||||
if mode not in ("subreddit", "username"):
|
||||
raise ValueError(
|
||||
"mode not correct, please enter 'username' or 'subreddit' as mode"
|
||||
)
|
||||
categories = data.get("categories", ["new", "hot"])
|
||||
search_queries = data.get("search_queries")
|
||||
number_posts = data.get("number_posts", 10)
|
||||
|
||||
import praw
|
||||
|
||||
reddit = praw.Reddit(
|
||||
client_id=data.get("client_id"),
|
||||
client_secret=data.get("client_secret"),
|
||||
user_agent=data.get("user_agent"),
|
||||
)
|
||||
|
||||
documents: List[Document] = []
|
||||
for search_query in search_queries:
|
||||
for category in categories:
|
||||
if mode == "subreddit":
|
||||
listing = reddit.subreddit(search_query)
|
||||
else:
|
||||
listing = reddit.redditor(search_query).submissions
|
||||
documents.extend(
|
||||
self._posts_to_documents(
|
||||
getattr(listing, category)(limit=number_posts), category
|
||||
)
|
||||
)
|
||||
return documents
|
||||
|
||||
@staticmethod
|
||||
def _posts_to_documents(posts: Iterable[Any], category: str) -> List[Document]:
|
||||
"""Convert praw submissions into documents."""
|
||||
documents = []
|
||||
for post in posts:
|
||||
documents.append(
|
||||
Document(
|
||||
text=post.selftext,
|
||||
doc_id=post.id,
|
||||
extra_info={
|
||||
"post_subreddit": post.subreddit_name_prefixed,
|
||||
"post_category": category,
|
||||
"title": post.title,
|
||||
"post_score": post.score,
|
||||
"post_id": post.id,
|
||||
"source": post.url,
|
||||
"post_author": str(post.author),
|
||||
},
|
||||
)
|
||||
)
|
||||
return documents
|
||||
Reference in new issue
Block a user