feat: fields to handle reddit loader + minor changes

This commit is contained in:
Siddhant Rai committed 2024-03-26 16:07:44 +05:30
1 parent 0c3970a266
commit eed1bfbe50
3 files changed
+120 -38

No files matched your search

+1 -1
View File
@@ -149,7 +149,7 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
}
def remote_worker(self, source_data, name_job, user, directory="temp", loader="url"):
def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
# sample = False
token_check = True
min_tokens = 150