Pg 4 (#2390)

* feat: postgres tests

* feat: mongo cutoff

* feat: mongo cutoff

* feat: adjust docs and compose files

* fix: mini code mongo removals

* fix: tests and k8s mongo stuff

* feat: test fixes

* fix: ruff

* fix: vale

* Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information'

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* fix: mini suggestions

* vale lint fix 2

* fix: codeql columns thing

* fix: test mongo

* fix: tests coverage

* feat: better tests 4

* feat: more tests

* feat: decent coverage

* fix: ruff fixes

* fix: remove mongo mock

* feat: enhance workflow engine and API routes; add document retrieval and source handling

* feat: e2e tests

* fix: mcp, mongo and more

* fix: mini codeql warning

* fix: agent chunk view

* fix: mini issues

* fix: more pg fixes

* feat: postgres prep on start

* feat: qa tests

* fix: mini improvements

* fix: tests

---------

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
Co-authored-by: Siddhant Rai <siddhant.rai.5686@gmail.com>

deployment/docker-compose-azure.yaml

@@ -17,7 +17,7 @@ services:
       # Override URLs to use docker service names
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
-      - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
     ports:
       - "7091:7091"
     volumes:
@@ -25,8 +25,10 @@ services:
       - ../application/inputs:/app/application/inputs
       - ../application/vectors:/app/application/vectors
     depends_on:
-        - redis
-        - mongo
+      redis:
+        condition: service_started
+      postgres:
+        condition: service_healthy
 
   worker:
     build: ../application
@@ -37,25 +39,34 @@ services:
       # Override URLs to use docker service names
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
-      - MONGO_URI=mongodb://mongo:27017/docsgpt
       - API_URL=http://backend:7091
+      - POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
     depends_on:
-        - redis
-        - mongo
+      redis:
+        condition: service_started
+      postgres:
+        condition: service_healthy
 
   redis:
     image: redis:6-alpine
     ports:
       - 6379:6379
 
-  mongo:
-    image: mongo:6
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      - POSTGRES_USER=docsgpt
+      - POSTGRES_PASSWORD=docsgpt
+      - POSTGRES_DB=docsgpt
     ports:
-      - 27017:27017
+      - "5432:5432"
     volumes:
-      - mongodb_data_container:/data/db
-
-
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U docsgpt -d docsgpt"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
 
 volumes:
-  mongodb_data_container:
+  postgres_data:

deployment/docker-compose-dev.yaml

@@ -6,14 +6,21 @@ services:
     ports:
       - 6379:6379
 
-  mongo:
-    image: mongo:6
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      - POSTGRES_USER=docsgpt
+      - POSTGRES_PASSWORD=docsgpt
+      - POSTGRES_DB=docsgpt
     ports:
-      - 27017:27017
+      - "5432:5432"
     volumes:
-      - mongodb_data_container:/data/db
-
-
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U docsgpt -d docsgpt"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
 
 volumes:
-  mongodb_data_container:
+  postgres_data:

deployment/docker-compose-hub.yaml

@@ -21,8 +21,8 @@ services:
     environment:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
-      - MONGO_URI=mongodb://mongo:27017/docsgpt
       - CACHE_REDIS_URL=redis://redis:6379/2
+      - POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
     ports:
       - "7091:7091"
     volumes:
@@ -30,8 +30,10 @@ services:
       - ../application/inputs:/app/inputs
       - ../application/vectors:/app/vectors
     depends_on:
-      - redis
-      - mongo
+      redis:
+        condition: service_started
+      postgres:
+        condition: service_healthy
 
 
   worker:
@@ -43,28 +45,39 @@ services:
     environment:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
-      - MONGO_URI=mongodb://mongo:27017/docsgpt
       - API_URL=http://backend:7091
       - CACHE_REDIS_URL=redis://redis:6379/2
+      - POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
     volumes:
       - ../application/indexes:/app/indexes
       - ../application/inputs:/app/inputs
       - ../application/vectors:/app/vectors
     depends_on:
-      - redis
-      - mongo
+      redis:
+        condition: service_started
+      postgres:
+        condition: service_healthy
 
   redis:
     image: redis:6-alpine
     ports:
       - 6379:6379
 
-  mongo:
-    image: mongo:6
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      - POSTGRES_USER=docsgpt
+      - POSTGRES_PASSWORD=docsgpt
+      - POSTGRES_DB=docsgpt
     ports:
-      - 27017:27017
+      - "5432:5432"
     volumes:
-      - mongodb_data_container:/data/db
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U docsgpt -d docsgpt"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
 
 volumes:
-  mongodb_data_container:
+  postgres_data:

deployment/docker-compose-local.yaml

@@ -15,12 +15,21 @@ services:
     ports:
       - 6379:6379
 
-  mongo:
-    image: mongo:6
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      - POSTGRES_USER=docsgpt
+      - POSTGRES_PASSWORD=docsgpt
+      - POSTGRES_DB=docsgpt
     ports:
-      - 27017:27017
+      - "5432:5432"
     volumes:
-      - mongodb_data_container:/data/db
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U docsgpt -d docsgpt"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
 
 volumes:
-  mongodb_data_container:
+  postgres_data:

deployment/docker-compose.yaml

@@ -22,8 +22,8 @@ services:
       # Override URLs to use docker service names
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
-      - MONGO_URI=mongodb://mongo:27017/docsgpt
       - CACHE_REDIS_URL=redis://redis:6379/2
+      - POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
     ports:
       - "7091:7091"
     volumes:
@@ -31,8 +31,10 @@ services:
       - ../application/inputs:/app/inputs
       - ../application/vectors:/app/vectors
     depends_on:
-      - redis
-      - mongo
+      redis:
+        condition: service_started
+      postgres:
+        condition: service_healthy
 
   worker:
     user: root
@@ -44,28 +46,40 @@ services:
       # Override URLs to use docker service names
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
-      - MONGO_URI=mongodb://mongo:27017/docsgpt
       - API_URL=http://backend:7091
       - CACHE_REDIS_URL=redis://redis:6379/2
+      - POSTGRES_URI=postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
     volumes:
       - ../application/indexes:/app/indexes
       - ../application/inputs:/app/inputs
       - ../application/vectors:/app/vectors
     depends_on:
-      - redis
-      - mongo
+      redis:
+        condition: service_started
+      postgres:
+        condition: service_healthy
 
   redis:
     image: redis:6-alpine
     ports:
       - 6379:6379
 
-  mongo:
-    image: mongo:6
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      - POSTGRES_USER=docsgpt
+      - POSTGRES_PASSWORD=docsgpt
+      - POSTGRES_DB=docsgpt
     ports:
-      - 27017:27017
+      - "5432:5432"
     volumes:
-      - mongodb_data_container:/data/db
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U docsgpt -d docsgpt"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
 
 volumes:
-  mongodb_data_container:
+  postgres_data:
+

deployment/k8s/deployments/docsgpt-deploy.yaml

@@ -12,6 +12,19 @@ spec:
       labels:
         app: docsgpt-api
     spec:
+      initContainers:
+      # Block pod start until Postgres accepts connections. The `postgres-init`
+      # Job is responsible for running alembic migrations; this container only
+      # waits for the server to be reachable.
+      - name: wait-for-postgres
+        image: postgres:16-alpine
+        command:
+          - sh
+          - -c
+          - |
+            until pg_isready -h postgres -p 5432 -U docsgpt -d docsgpt; do
+              echo "Waiting for postgres..."; sleep 2;
+            done
       containers:
       - name: docsgpt-api
         image: arc53/docsgpt
@@ -32,6 +45,18 @@ spec:
           value: "application/app.py"
         - name: DEPLOYMENT_TYPE
           value: "cloud"
+        - name: POSTGRES_URI
+          valueFrom:
+            secretKeyRef:
+              name: docsgpt-secrets
+              key: POSTGRES_URI
+        # Disable in-app auto-bootstrap. The `postgres-init` Job under
+        # deployment/k8s/jobs/ owns schema creation and Alembic migrations,
+        # so application pods must not race with it on rollout.
+        - name: AUTO_MIGRATE
+          value: "false"
+        - name: AUTO_CREATE_DB
+          value: "false"
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -47,6 +72,16 @@ spec:
       labels:
         app: docsgpt-worker
     spec:
+      initContainers:
+      - name: wait-for-postgres
+        image: postgres:16-alpine
+        command:
+          - sh
+          - -c
+          - |
+            until pg_isready -h postgres -p 5432 -U docsgpt -d docsgpt; do
+              echo "Waiting for postgres..."; sleep 2;
+            done
       containers:
       - name: docsgpt-worker
         image: arc53/docsgpt
@@ -64,6 +99,18 @@ spec:
         env:
         - name: API_URL
           value: "http://<your-api-endpoint>"
+        - name: POSTGRES_URI
+          valueFrom:
+            secretKeyRef:
+              name: docsgpt-secrets
+              key: POSTGRES_URI
+        # Disable in-app auto-bootstrap. The `postgres-init` Job under
+        # deployment/k8s/jobs/ owns schema creation and Alembic migrations,
+        # so application pods must not race with it on rollout.
+        - name: AUTO_MIGRATE
+          value: "false"
+        - name: AUTO_CREATE_DB
+          value: "false"
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -95,4 +142,4 @@ spec:
         - name: VITE_API_HOST
           value: "http://<your-api-endpoint>"
         - name: VITE_API_STREAMING
-          value: "true"
+          value: "true"

deployment/k8s/deployments/postgres-deploy.yaml

@@ -0,0 +1,79 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgres-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi  # Adjust size as needed
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postgres
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      containers:
+      - name: postgres
+        image: postgres:16-alpine
+        ports:
+        - containerPort: 5432
+        env:
+        - name: POSTGRES_USER
+          value: "docsgpt"
+        - name: POSTGRES_PASSWORD
+          value: "docsgpt"
+        - name: POSTGRES_DB
+          value: "docsgpt"
+        - name: PGDATA
+          value: "/var/lib/postgresql/data/pgdata"
+        resources:
+          limits:
+            memory: "1Gi"
+            cpu: "1"
+          requests:
+            memory: "256Mi"
+            cpu: "100m"
+        volumeMounts:
+        - name: postgres-data
+          mountPath: /var/lib/postgresql/data
+        readinessProbe:
+          exec:
+            command:
+              - pg_isready
+              - -U
+              - docsgpt
+              - -d
+              - docsgpt
+          initialDelaySeconds: 5
+          periodSeconds: 5
+          timeoutSeconds: 3
+          failureThreshold: 6
+        livenessProbe:
+          exec:
+            command:
+              - pg_isready
+              - -U
+              - docsgpt
+              - -d
+              - docsgpt
+          initialDelaySeconds: 30
+          periodSeconds: 15
+          timeoutSeconds: 5
+          failureThreshold: 3
+      volumes:
+      - name: postgres-data
+        persistentVolumeClaim:
+          claimName: postgres-pvc

deployment/k8s/docsgpt-secrets.yaml

@@ -3,6 +3,15 @@ kind: Secret
 metadata:
   name: docsgpt-secrets
 type: Opaque
+# Notes:
+# - POSTGRES_URI below decodes to:
+#     postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
+#   This matches the default Postgres Deployment/Service in this kustomization
+#   and mirrors the compose-level default (see deployment/docker-compose.yaml).
+# - If you still need the MongoDB-backed vector store (VECTOR_STORE=mongodb),
+#   manually add a MONGO_URI key below (base64-encoded) and apply the
+#   opt-in manifests under deployment/k8s/optional-mongo/. Example:
+#     MONGO_URI: <base64 of mongodb://mongodb-service:27017/docsgpt?retryWrites=true&w=majority>
 data:
   LLM_PROVIDER: ZG9jc2dwdA==
   INTERNAL_KEY: aW50ZXJuYWw=
@@ -10,6 +19,7 @@ data:
   CELERY_RESULT_BACKEND: cmVkaXM6Ly9yZWRpcy1zZXJ2aWNlOjYzNzkvMA==
   QDRANT_URL: cmVkaXM6Ly9yZWRpcy1zZXJ2aWNlOjYzNzkvMA==
   QDRANT_PORT: NjM3OQ==
-  MONGO_URI: bW9uZ29kYjovL21vbmdvZGItc2VydmljZToyNzAxNy9kb2NzZ3B0P3JldHJ5V3JpdGVzPXRydWUmdz1tYWpvcml0eQ==
-  mongo-user: bW9uZ28tdXNlcg==
-  mongo-password: bW9uZ28tcGFzc3dvcmQ=
+  # postgresql://docsgpt:docsgpt@postgres:5432/docsgpt
+  POSTGRES_URI: cG9zdGdyZXNxbDovL2RvY3NncHQ6ZG9jc2dwdEBwb3N0Z3Jlczo1NDMyL2RvY3NncHQ=
+  postgres-user: ZG9jc2dwdA==
+  postgres-password: ZG9jc2dwdA==

deployment/k8s/jobs/postgres-init-job.yaml

@@ -0,0 +1,45 @@
+# One-shot migrator: runs `python scripts/db/init_postgres.py` (alembic
+# upgrade head), then exits. The backend and worker Deployments rely on
+# an init container that blocks until Postgres is reachable, but this Job
+# ensures the schema is migrated before application pods start serving.
+#
+# Re-apply after upgrades to run new migrations. Safe to re-run: idempotent.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: postgres-init
+spec:
+  backoffLimit: 6
+  template:
+    metadata:
+      labels:
+        app: postgres-init
+    spec:
+      restartPolicy: OnFailure
+      initContainers:
+      - name: wait-for-postgres
+        image: postgres:16-alpine
+        command:
+          - sh
+          - -c
+          - |
+            until pg_isready -h postgres -p 5432 -U docsgpt -d docsgpt; do
+              echo "Waiting for postgres..."; sleep 2;
+            done
+      containers:
+      - name: postgres-init
+        image: arc53/docsgpt
+        command: ["python", "scripts/db/init_postgres.py"]
+        envFrom:
+        - secretRef:
+            name: docsgpt-secrets
+        env:
+        - name: FLASK_APP
+          value: "application/app.py"
+        resources:
+          limits:
+            memory: "1Gi"
+            cpu: "1"
+          requests:
+            memory: "256Mi"
+            cpu: "100m"

deployment/k8s/kustomization.yaml

@@ -0,0 +1,20 @@
+# Default DocsGPT Kubernetes stack.
+#
+# Uses Postgres as the user-data store (mirrors
+# deployment/docker-compose.yaml). MongoDB manifests are opt-in and live
+# under deployment/k8s/optional-mongo/ — they are NOT included here and
+# will not be applied by `kubectl apply -k deployment/k8s/`.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - docsgpt-secrets.yaml
+  - deployments/postgres-deploy.yaml
+  - deployments/qdrant-deploy.yaml
+  - deployments/redis-deploy.yaml
+  - deployments/docsgpt-deploy.yaml
+  - services/postgres-service.yaml
+  - services/qdrant-service.yaml
+  - services/redis-service.yaml
+  - services/docsgpt-service.yaml
+  - jobs/postgres-init-job.yaml

deployment/k8s/optional-mongo/README.md

@@ -0,0 +1,23 @@
+# Optional: MongoDB manifests
+
+These manifests are **opt-in**. The default DocsGPT install uses Postgres
+for user data (see `deployment/k8s/deployments/postgres-deploy.yaml`).
+
+Apply the manifests in this directory only if you run DocsGPT with the
+MongoDB-backed vector store (`VECTOR_STORE=mongodb`) and need an
+in-cluster MongoDB, or if you are intentionally running on the legacy
+MongoDB user-data store during the Postgres migration window.
+
+Mirrors `deployment/optional/` for compose — not applied by the default
+`kubectl apply -k deployment/k8s/`.
+
+## Usage
+
+```bash
+kubectl apply -f deployment/k8s/optional-mongo/deployments/mongo-deploy.yaml
+kubectl apply -f deployment/k8s/optional-mongo/services/mongo-service.yaml
+```
+
+Then extend `docsgpt-secrets.yaml` with a base64-encoded `MONGO_URI`
+pointing at `mongodb://mongodb-service:27017/docsgpt?retryWrites=true&w=majority`
+(or your Atlas/external URI) before re-applying the secret.

deployment/k8s/services/postgres-service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+spec:
+  selector:
+    app: postgres
+  ports:
+    - protocol: TCP
+      port: 5432
+      targetPort: 5432
+  type: ClusterIP