diff --git a/application/api/user/routes.py b/application/api/user/routes.py index b25e587f..231ad9a4 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -3,6 +3,7 @@ import math import os import shutil import uuid +import json from bson.binary import Binary, UuidRepresentation from bson.dbref import DBRef @@ -428,18 +429,22 @@ class UploadRemote(Resource): return missing_fields try: - if "repo_url" in data: - source_data = data["repo_url"] - loader = "github" - else: - source_data = data["data"] - loader = data["source"] + config = json.loads(data["data"]) + source_data = None - task = ingest_remote.delay( + if data["source"] == "github": + source_data = config.get("repo_url") + elif data["source"] in ["crawler", "url"]: + source_data = config.get("url") + elif data["source"] == "reddit": + source_data = config + + + task = ingest_remote.delay( source_data=source_data, job_name=data["name"], user=data["user"], - loader=loader, + loader=data["source"] ) except Exception as err: current_app.logger.error(f"Error uploading remote source: {err}") diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 84ba0eb0..d70a202f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1635,7 +1635,7 @@ "version": "18.3.0", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.0.tgz", "integrity": "sha512-EhwApuTmMBmXuFOikhQLIBUn6uFg81SwLMOAUgodJF14SOBOCMdU04gDoYi0WOJJHD144TL32z4yDqCW3dnkQg==", - "devOptional": true, + "dev": true, "dependencies": { "@types/react": "*" } @@ -9376,7 +9376,7 @@ "version": "5.7.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.2.tgz", "integrity": "sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==", - "devOptional": true, + "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/frontend/src/components/Input.tsx b/frontend/src/components/Input.tsx index f7b1765e..f224e2ba 100644 --- a/frontend/src/components/Input.tsx +++ b/frontend/src/components/Input.tsx @@ -8,6 +8,7 @@ const Input = ({ isAutoFocused = false, placeholder, label, + required = false, maxLength, className, colorVariant = 'silver', @@ -40,13 +41,19 @@ const Input = ({ onChange={onChange} onPaste={onPaste} onKeyDown={onKeyDown} + required={required} > {children} {label && (
- + {label} + {required && ( + + * + + )}
)} diff --git a/frontend/src/components/types/index.ts b/frontend/src/components/types/index.ts index a7ff5405..1d4b3a61 100644 --- a/frontend/src/components/types/index.ts +++ b/frontend/src/components/types/index.ts @@ -9,6 +9,7 @@ export type InputProps = { name?: string; placeholder?: string; label?: string; + required?: boolean; className?: string; children?: React.ReactElement; onChange: ( diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index 35058be0..df7895ed 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -156,7 +156,9 @@ "completed": "Training completed", "wait": "This may take several minutes", "tokenLimit": "Over the token limit, please consider uploading smaller document" - } + }, + "showAdvanced": "Show advanced options", + "hideAdvanced": "Hide advanced options" }, "createAPIKey": { "label": "Create New API Key", diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index d0b47874..123f9fdd 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -156,7 +156,9 @@ "completed": "Entrenamiento completado", "wait": "Esto puede tardar varios minutos", "tokenLimit": "Excede el límite de tokens, considere cargar un documento más pequeño" - } + }, + "showAdvanced": "Mostrar opciones avanzadas", + "hideAdvanced": "Ocultar opciones avanzadas" }, "createAPIKey": { "label": "Crear Nueva Clave de API", diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index 2fdc2c61..b4cd1531 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -155,7 +155,9 @@ "completed": "トレーニング完了", "wait": "数分かかる場合があります", "tokenLimit": "トークン制限を超えています。より小さいドキュメントをアップロードしてください" - } + }, + "showAdvanced": "詳細オプションを表示", + "hideAdvanced": "詳細オプションを非表示" }, "createAPIKey": { "label": "新しいAPIキーを作成", diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index 087b8dfd..d6bc2bdf 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -156,7 +156,9 @@ "completed": "Обучение завершено", "wait": "Это может занять несколько минут", "tokenLimit": "Превышен лимит токенов, рассмотрите возможность загрузки документа меньшего размера" - } + }, + "showAdvanced": "Показать расширенные настройки", + "hideAdvanced": "Скрыть расширенные настройки" }, "createAPIKey": { "label": "Создать новый API ключ", diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index f2abf1e5..8eef9e56 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -156,7 +156,9 @@ "completed": "訓練完成", "wait": "這可能需要幾分鐘", "tokenLimit": "超出令牌限制，請考慮上傳較小的文檔" - } + }, + "showAdvanced": "顯示進階選項", + "hideAdvanced": "隱藏進階選項" }, "createAPIKey": { "label": "建立新的 API 金鑰", diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index 56c8fbb1..eb9d1e21 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -156,7 +156,9 @@ "completed": "训练完成", "wait": "这可能需要几分钟", "tokenLimit": "超出令牌限制，请考虑上传较小的文档" - } + }, + "showAdvanced": "显示高级选项", + "hideAdvanced": "隐藏高级选项" }, "createAPIKey": { "label": "创建新的 API 密钥", diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 725ecdb1..0e030beb 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -20,25 +20,11 @@ import WrapperModal from '../modals/WrapperModal'; import { IngestorType, IngestorConfig, - RedditIngestorConfig, - GithubIngestorConfig, - CrawlerIngestorConfig, - UrlIngestorConfig, IngestorFormSchemas, FormField, } from './types/ingestor'; import { IngestorDefaultConfigs } from '../upload/types/ingestor'; -type IngestorState = { - type: IngestorType; - name: string; - config: - | RedditIngestorConfig - | GithubIngestorConfig - | CrawlerIngestorConfig - | UrlIngestorConfig; -}; - function Upload({ receivedFile = [], setModalState, @@ -55,93 +41,124 @@ function Upload({ onSuccessfulUpload?: () => void; }) { const [docName, setDocName] = useState(receivedFile[0]?.name); + const [remoteName, setRemoteName] = useState(''); const [files, setfiles] = useState(receivedFile); const [activeTab, setActiveTab] = useState(renderTab); + const [showAdvancedOptions, setShowAdvancedOptions] = useState(false); const renderFormFields = () => { const schema = IngestorFormSchemas[ingestor.type]; + if (!schema) return null; - return schema.map((field: FormField) => { - switch (field.type) { - case 'string': - return ( -
- - handleIngestorChange(field.name, e.target.value) - } - borderVariant="thin" - label={field.label} - colorVariant="gray" - /> -
- ); - case 'number': - return ( -
- - handleIngestorChange(field.name, parseInt(e.target.value)) - } - borderVariant="thin" - label={field.label} - colorVariant="gray" - /> -
- ); - case 'enum': - return ( -
- { - const value = - typeof selected === 'string' ? selected : selected.value; - handleIngestorChange(field.name, value); - }} - size="w-full" - rounded="3xl" - placeholder={field.label} - border="border" - borderColor="gray-5000" - /> -
- ); - case 'boolean': - return ( -
- { - const syntheticEvent = { - target: { - name: field.name, - value: checked, - }, - } as unknown as React.ChangeEvent; - handleIngestorChange(field.name, syntheticEvent.target.value); - }} - className="mt-2" - /> -
- ); - default: - return null; - } - }); + const generalFields = schema.filter((field) => !field.advanced); + const advancedFields = schema.filter((field) => field.advanced); + + return ( + <> + {generalFields.map((field: FormField) => renderField(field))} + + {advancedFields.length > 0 && showAdvancedOptions && ( + <> +
+ {advancedFields.map((field: FormField) => renderField(field))} + + )} + + ); + }; + + const renderField = (field: FormField) => { + const isRequired = field.required ?? false; + switch (field.type) { + case 'string': + return ( + + handleIngestorChange( + field.name as keyof IngestorConfig['config'], + e.target.value, + ) + } + borderVariant="thin" + label={field.label} + required={isRequired} + colorVariant="gray" + /> + ); + case 'number': + return ( + + handleIngestorChange( + field.name as keyof IngestorConfig['config'], + Number(e.target.value), + ) + } + borderVariant="thin" + label={field.label} + required={isRequired} + colorVariant="gray" + /> + ); + case 'enum': + return ( + + opt.value === + ingestor.config[field.name as keyof typeof ingestor.config], + ) || null + } + onSelect={(selected: { label: string; value: string }) => { + handleIngestorChange( + field.name as keyof IngestorConfig['config'], + selected.value, + ); + }} + size="w-full" + rounded="3xl" + placeholder={field.label} + border="border" + borderColor="gray-5000" + /> + ); + case 'boolean': + return ( + { + handleIngestorChange( + field.name as keyof IngestorConfig['config'], + checked, + ); + }} + className="mt-2" + /> + ); + default: + return null; + } }; // New unified ingestor state @@ -172,11 +189,6 @@ function Upload({ { label: 'Reddit', value: 'reddit' }, ]; - const [urlType, setUrlType] = useState<{ label: string; value: string }>({ - label: 'Crawler', - value: 'crawler', - }); - const sourceDocs = useSelector(selectSourceDocs); useEffect(() => { if (setTimeoutRef.current) { @@ -363,7 +375,7 @@ function Upload({ const onDrop = useCallback((acceptedFiles: File[]) => { setfiles(acceptedFiles); - setDocName(acceptedFiles[0]?.name); + setDocName(acceptedFiles[0]?.name || ''); }, []); const doNothing = () => undefined; @@ -374,7 +386,7 @@ function Upload({ formData.append('file', file); }); - formData.append('name', activeTab === 'file' ? docName : ingestor.name); + formData.append('name', docName); formData.append('user', 'local'); const apiHost = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest(); @@ -394,43 +406,58 @@ function Upload({ const uploadRemote = () => { const formData = new FormData(); - formData.append('name', ingestor.name); + formData.append('name', remoteName); formData.append('user', 'local'); formData.append('source', ingestor.type); - if (ingestor.type === 'reddit') { - const redditConfig = ingestor.config as RedditIngestorConfig; - redditConfig.name = ingestor.name; - formData.set('data', JSON.stringify(redditConfig)); - } else if (ingestor.type === 'github') { - const githubConfig = ingestor.config as GithubIngestorConfig; - githubConfig.name = ingestor.name; - formData.append('repo_url', githubConfig.repo_url); - formData.append('data', githubConfig.repo_url); - } else { - const urlBasedConfig = ingestor.config as - | CrawlerIngestorConfig - | UrlIngestorConfig; - urlBasedConfig.name = ingestor.name; - formData.append('data', urlBasedConfig.url); - } + const defaultConfig = IngestorDefaultConfigs[ingestor.type].config; - const apiHost = import.meta.env.VITE_API_HOST; + const mergedConfig = { ...defaultConfig, ...ingestor.config }; + const filteredConfig = Object.entries(mergedConfig).reduce( + (acc, [key, value]) => { + const field = IngestorFormSchemas[ingestor.type].find( + (f) => f.name === key, + ); + // Include the field if: + // 1. It's required, or + // 2. It's optional and has a non-empty value + if ( + field?.required || + (value !== undefined && value !== null && value !== '') + ) { + acc[key] = value; + } + return acc; + }, + {} as Record, + ); + + formData.append('data', JSON.stringify(filteredConfig)); + + const apiHost: string = import.meta.env.VITE_API_HOST; const xhr = new XMLHttpRequest(); - xhr.upload.addEventListener('progress', (event) => { - const progress = +((event.loaded / event.total) * 100).toFixed(2); - setProgress({ type: 'UPLOAD', percentage: progress }); + xhr.upload.addEventListener('progress', (event: ProgressEvent) => { + if (event.lengthComputable) { + const progressPercentage = +( + (event.loaded / event.total) * + 100 + ).toFixed(2); + setProgress({ type: 'UPLOAD', percentage: progressPercentage }); + } }); xhr.onload = () => { - const { task_id } = JSON.parse(xhr.responseText); - setTimeoutRef.current = setTimeout(() => { - setProgress({ type: 'TRAINING', percentage: 0, taskId: task_id }); + const response = JSON.parse(xhr.responseText) as { task_id: string }; + setTimeoutRef.current = window.setTimeout(() => { + setProgress({ + type: 'TRAINING', + percentage: 0, + taskId: response.task_id, + }); }, 3000); }; - xhr.open('POST', `${apiHost + '/api/remote'}`); + xhr.open('POST', `${apiHost}/api/remote`); xhr.send(formData); }; - const { getRootProps, getInputProps, isDragActive } = useDropzone({ onDrop, multiple: true, @@ -461,43 +488,55 @@ function Upload({ }, }); - const isUploadDisabled = () => { + const isUploadDisabled = (): boolean => { if (activeTab === 'file') { - return !docName || files.length === 0; + return !docName?.trim() || files.length === 0; } - - if (activeTab !== 'remote') return false; - - if (!ingestor.name) return true; - - return Object.values(ingestor.config).some((value) => { - if (Array.isArray(value)) { - return value.length === 0; + if (activeTab === 'remote') { + if (!remoteName?.trim()) { + return true; } - return !value; - }); - }; + const formFields: FormField[] = IngestorFormSchemas[ingestor.type]; + for (const field of formFields) { + if (field.required) { + // Validate only required fields + const value = + ingestor.config[field.name as keyof typeof ingestor.config]; - const handleIngestorChange = (key: string, value: any) => { - setIngestor((prevState: IngestorConfig): IngestorConfig => { - if (key === 'name') { - return { - ...prevState, - name: value, - }; + if (typeof value === 'string' && !value.trim()) { + return true; + } + + if ( + typeof value === 'number' && + (value === null || value === undefined || value <= 0) + ) { + return true; + } + + if (typeof value === 'boolean' && value === undefined) { + return true; + } + } } - - return { - ...prevState, - config: { - ...(prevState.config as any), - [key]: value, - }, - }; - }); + return false; + } + return true; + }; + const handleIngestorChange = ( + key: keyof IngestorConfig['config'], + value: string | number | boolean, + ) => { + setIngestor((prevState) => ({ + ...prevState, + config: { + ...prevState.config, + [key]: value, + }, + })); }; - const handleIngestorTypeChange = (type: IngestorType) => { + //Updates the ingestor seleced in dropdown and resets the config to the default config for that type const defaultConfig = IngestorDefaultConfigs[type]; setIngestor({ @@ -598,7 +637,7 @@ function Upload({ {activeTab === 'remote' && ( <> opt.value === ingestor.type) || null @@ -614,15 +653,26 @@ function Upload({ - setIngestor({ ...ingestor, name: e.target.value }) - } + value={remoteName} + onChange={(e) => setRemoteName(e.target.value)} borderVariant="thin" placeholder="Name" label="Name" + required={true} /> {renderFormFields()} + {IngestorFormSchemas[ingestor.type].some( + (field) => field.advanced, + ) && ( + + )} )}
@@ -643,6 +693,7 @@ function Upload({ uploadRemote(); } }} + disabled={isUploadDisabled()} className={`rounded-3xl px-4 py-2 font-medium ${ isUploadDisabled() ? 'cursor-not-allowed bg-gray-300 text-gray-500' diff --git a/frontend/src/upload/types/ingestor.ts b/frontend/src/upload/types/ingestor.ts index 83c74bfd..cd709847 100644 --- a/frontend/src/upload/types/ingestor.ts +++ b/frontend/src/upload/types/ingestor.ts @@ -1,5 +1,5 @@ export interface BaseIngestorConfig { - name: string; + [key: string]: string | number | boolean | undefined; } export interface RedditIngestorConfig extends BaseIngestorConfig { @@ -44,9 +44,11 @@ export type IngestorFormData = { export type FieldType = 'string' | 'number' | 'enum' | 'boolean'; export interface FormField { - name: keyof BaseIngestorConfig | string; + name: string; label: string; type: FieldType; + required?: boolean; + advanced?: boolean; options?: { label: string; value: string }[]; } @@ -56,6 +58,7 @@ export const IngestorFormSchemas: Record = { name: 'url', label: 'URL', type: 'string', + required: true, }, ], url: [ @@ -63,6 +66,7 @@ export const IngestorFormSchemas: Record = { name: 'url', label: 'URL', type: 'string', + required: true, }, ], reddit: [ @@ -70,26 +74,31 @@ export const IngestorFormSchemas: Record = { name: 'client_id', label: 'Client ID', type: 'string', + required: true, }, { name: 'client_secret', label: 'Client Secret', type: 'string', + required: true, }, { name: 'user_agent', label: 'User Agent', type: 'string', + required: true, }, { name: 'search_queries', label: 'Search Queries', type: 'string', + required: true, }, { name: 'number_posts', label: 'Number of Posts', type: 'number', + required: true, }, ], github: [ @@ -97,6 +106,7 @@ export const IngestorFormSchemas: Record = { name: 'repo_url', label: 'Repository URL', type: 'string', + required: true, }, ], };