From 8291a3408e77b59b70c846c4c72f56b04ba3365a Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Sun, 4 Oct 2026 23:08:09 +0700 Subject: [PATCH] feat(litellm): add LiteLLM proxy with PostgreSQL and Redis --- README.md | 1 + litellm/.env.example | 41 ++++++++++++ litellm/README.md | 79 ++++++++++++++++++++++ litellm/compose.yml | 86 ++++++++++++++++++++++++ litellm/config.yaml | 156 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 363 insertions(+) create mode 100644 litellm/.env.example create mode 100644 litellm/README.md create mode 100644 litellm/compose.yml create mode 100644 litellm/config.yaml diff --git a/README.md b/README.md index bfcc434..8df6113 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ Each links to its own README for variables, ports, and storage. | [diun](diun/README.md) | Image-update notifier, reading the Docker API through a read-only proxy | | [gitea-mirror](gitea-mirror/README.md) | Gitea + PostgreSQL + gitea-mirror, mirroring GitHub repos | | [goclaw](goclaw/README.md) | Multi-tenant AI agent gateway, with pgvector PostgreSQL | +| [litellm](litellm/README.md) | LiteLLM proxy in front of many LLM providers, with PostgreSQL and Redis | | [opencode](opencode/README.md) | opencode coding agent, served as a browser UI | | [paseo](paseo/README.md) | Paseo coding-agent daemon and web UI | | [traffmonetizer](traffmonetizer/README.md) | TraffMonetizer bandwidth-sharing client | diff --git a/litellm/.env.example b/litellm/.env.example new file mode 100644 index 0000000..2a506e8 --- /dev/null +++ b/litellm/.env.example @@ -0,0 +1,41 @@ +# Copy to .env and fill in. Never commit .env. +# +# cp .env.example .env + +# Admin key for the proxy API and the virtual keys it issues. Must start with sk-. +# Generate one with: echo "sk-$(openssl rand -hex 24)" +LITELLM_MASTER_KEY= + +# Admin UI login. +UI_USERNAME=admin +UI_PASSWORD= + +# PostgreSQL credentials, used by the database and by LiteLLM's DATABASE_URL. +POSTGRES_USER=litellm +POSTGRES_PASSWORD= +POSTGRES_DB=litellm + +# OpenRouter backs openrouter-free, the target of every model_group_alias in +# config.yaml. Gemini backs gemini-embedding-001 and the Gemini chat models. +OPENROUTER_API_KEY= +GEMINI_API_KEY= + +# Keep models added in the admin UI in PostgreSQL. +STORE_MODEL_IN_DB=True + +LITELLM_MODE=PRODUCTION +LITELLM_LOG=ERROR + +# Provider keys for the other models in config.yaml. Uncomment here and in +# compose.yml to enable a provider; an unset key disables only its models. +# TOKENROUTER_API_KEY= +# ORCAROUTER_API_KEY= +# KIMI_API_KEY= +# MINIMAX_API_KEY= +# KILO_API_KEY= +# OPENCODE_API_KEY= +# NVIDIA_NIM_API_KEY= +# OLLAMA_API_KEY= +# Cloudflare Workers AI, for the commented-out Cloudflare model in config.yaml. +# CLOUDFLARE_API_KEY= +# CLOUDFLARE_ACCOUNT_ID= diff --git a/litellm/README.md b/litellm/README.md new file mode 100644 index 0000000..8811253 --- /dev/null +++ b/litellm/README.md @@ -0,0 +1,79 @@ +# litellm + +[LiteLLM](https://docs.litellm.ai) proxy: one OpenAI-compatible endpoint in +front of many LLM providers, with virtual keys, spend tracking and an admin UI +at `/ui`. + +Three containers: `litellm`, `postgres` for keys, teams, budgets and spend +logs, and `redis` for the response cache. + +## Setup + +1. Set `LITELLM_MASTER_KEY`, `UI_PASSWORD`, `POSTGRES_PASSWORD`, + `OPENROUTER_API_KEY` and `GEMINI_API_KEY`. Uncomment any other provider key + whose models you use, in both `compose.yml` and the environment. +2. Map the domain to port `4000` and deploy. Prisma migrations run on start. +3. Open `/ui` and log in with `UI_USERNAME` / `UI_PASSWORD`. + +Health check: `GET /health/liveliness`, also the compose healthcheck. + +## Models + +`config.yaml` holds the model list, the `model_group_alias` routing and the +proxy settings; it is mounted read-only at `/app/config.yaml`. Editing it +redeploys the service. Every provider key in it is an `os.environ/` reference, +so the file carries no secrets. + +Models can also be added in the admin UI. `STORE_MODEL_IN_DB` keeps those in +PostgreSQL, so they survive a redeploy without touching the file. + +## Environment + +| Variable | Default | Purpose | +| --- | --- | --- | +| `LITELLM_MASTER_KEY` | — | Admin API key; must start with `sk-` | +| `UI_USERNAME` / `UI_PASSWORD` | `admin` / — | Admin UI login | +| `POSTGRES_USER` / `POSTGRES_PASSWORD` / `POSTGRES_DB` | `litellm` / — / `litellm` | Database credentials, read by both containers | +| `OPENROUTER_API_KEY` | empty | OpenRouter models, including `openrouter-free`, the alias target | +| `GEMINI_API_KEY` | empty | Gemini chat models and `gemini-embedding-001` | +| `STORE_MODEL_IN_DB` | `True` | Keep UI-added models in PostgreSQL | +| `LITELLM_MODE` | `PRODUCTION` | Skips loading a `.env` inside the container | +| `LITELLM_LOG` | `ERROR` | Log level | +| `TOKENROUTER_API_KEY`, `ORCAROUTER_API_KEY`, `KIMI_API_KEY`, `MINIMAX_API_KEY`, `KILO_API_KEY`, `OPENCODE_API_KEY`, `NVIDIA_NIM_API_KEY`, `OLLAMA_API_KEY` | optional | Keys for the other providers in `config.yaml` | +| `CLOUDFLARE_API_KEY` / `CLOUDFLARE_ACCOUNT_ID` | optional | For the commented-out Cloudflare model | + +`LITELLM_MASTER_KEY` and `UI_PASSWORD` fail fast if unset: the master key is +the only thing between the domain and every provider key behind the proxy. + +`REDIS_HOST` and `REDIS_PORT` are fixed in `compose.yml`, as properties of this +layout. Redis runs without a password; it is reachable only on the stack's +own network. + +OpenRouter and Gemini stay active because the rest of this setup leans on +them: every `model_group_alias` routes to `openrouter-free`, and +`gemini-embedding-001` serves embeddings. The other providers are optional. An +unset key disables only the models that reference it — LiteLLM starts and +passes its health check with every provider key unset — so a provider is +enabled by uncommenting its line, without touching `config.yaml`. + +## Storage + +| Volume | Mount | Holds | +| --- | --- | --- | +| `pg-data` | `/var/lib/postgresql/data` | The database | +| `redis-data` | `/data` | Redis append-only file | + +## Images + +`ghcr.io/berriai/litellm-database:main-stable` is the variant with the Prisma +client for PostgreSQL built in. `main-stable` is upstream's moving stable tag. + +`postgres:16-alpine` and `redis:7-alpine` track their major versions. Moving +PostgreSQL to a new major needs a dump and restore; the data directory does +not carry across majors. + +## Resources + +`litellm` is capped at 4 GB of memory. Gunicorn runs two workers and recycles +each after 1000 requests, which bounds the slow memory growth of long-lived +LiteLLM workers. diff --git a/litellm/compose.yml b/litellm/compose.yml new file mode 100644 index 0000000..9bc52c2 --- /dev/null +++ b/litellm/compose.yml @@ -0,0 +1,86 @@ +services: + litellm: + image: ghcr.io/berriai/litellm-database:main-stable + restart: unless-stopped + environment: + - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY:?required} + - UI_USERNAME=${UI_USERNAME:-admin} + - UI_PASSWORD=${UI_PASSWORD:?required} + - DATABASE_URL=postgresql://${POSTGRES_USER:-litellm}:${POSTGRES_PASSWORD:?required}@postgres:5432/${POSTGRES_DB:-litellm} + - REDIS_HOST=redis + - REDIS_PORT=6379 + - OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-} + - GEMINI_API_KEY=${GEMINI_API_KEY:-} + - STORE_MODEL_IN_DB=${STORE_MODEL_IN_DB:-True} + - LITELLM_MODE=${LITELLM_MODE:-PRODUCTION} + - LITELLM_LOG=${LITELLM_LOG:-ERROR} + # - TOKENROUTER_API_KEY=${TOKENROUTER_API_KEY:-} + # - ORCAROUTER_API_KEY=${ORCAROUTER_API_KEY:-} + # - KIMI_API_KEY=${KIMI_API_KEY:-} + # - MINIMAX_API_KEY=${MINIMAX_API_KEY:-} + # - KILO_API_KEY=${KILO_API_KEY:-} + # - OPENCODE_API_KEY=${OPENCODE_API_KEY:-} + # - NVIDIA_NIM_API_KEY=${NVIDIA_NIM_API_KEY:-} + # - OLLAMA_API_KEY=${OLLAMA_API_KEY:-} + # - CLOUDFLARE_API_KEY=${CLOUDFLARE_API_KEY:-} + # - CLOUDFLARE_ACCOUNT_ID=${CLOUDFLARE_ACCOUNT_ID:-} + volumes: + - ./config.yaml:/app/config.yaml:ro + command: + - --config + - /app/config.yaml + - --port + - "4000" + - --run_gunicorn + - --num_workers + - "2" + - --max_requests_before_restart + - "1000" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request;urllib.request.urlopen('http://127.0.0.1:4000/health/liveliness',timeout=5)"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + deploy: + resources: + limits: + memory: 4G + + # PostgreSQL for keys, teams, budgets and spend logs. + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + - POSTGRES_USER=${POSTGRES_USER:-litellm} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?required} + - POSTGRES_DB=${POSTGRES_DB:-litellm} + volumes: + - pg-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -h localhost -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 3 + + # Redis for the response cache and router state. + redis: + image: redis:7-alpine + restart: unless-stopped + command: redis-server --appendonly yes + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 3 + +volumes: + pg-data: + redis-data: diff --git a/litellm/config.yaml b/litellm/config.yaml new file mode 100644 index 0000000..8d96570 --- /dev/null +++ b/litellm/config.yaml @@ -0,0 +1,156 @@ +model_list: + - model_name: kimi-k3-free + litellm_params: + model: openai/moonshotai/kimi-k3-free + api_base: https://api.tokenrouter.com/v1 + api_key: os.environ/TOKENROUTER_API_KEY + + - model_name: glm-5.3-flash-free + litellm_params: + model: openai/z-ai/glm-5.3-flash-free + api_base: https://api.orcarouter.ai/v1 + api_key: os.environ/ORCAROUTER_API_KEY + + - model_name: kimi-for-coding + litellm_params: + model: anthropic/kimi-for-coding + api_base: https://api.kimi.com/coding + api_key: os.environ/KIMI_API_KEY + + - model_name: MiniMax-M3 + litellm_params: + model: minimax/MiniMax-M3 + api_key: os.environ/MINIMAX_API_KEY + + - model_name: gemma-4-31b-it:free + litellm_params: + model: openrouter/google/gemma-4-31b-it:free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: openrouter-free + litellm_params: + model: openrouter/openrouter/free + api_key: os.environ/OPENROUTER_API_KEY + + - model_name: orcarouter-free + litellm_params: + model: openai/orcarouter/free + api_base: https://api.orcarouter.ai/v1 + api_key: os.environ/ORCAROUTER_API_KEY + + - model_name: kilo-free + litellm_params: + model: openai/kilo-auto/free + api_base: https://api.kilo.ai/api/gateway + api_key: os.environ/KILO_API_KEY + + - model_name: deepseek-v4-flash-free + litellm_params: + model: openai/deepseek-v4-flash-free + api_base: https://opencode.ai/zen/v1 + api_key: os.environ/OPENCODE_API_KEY + + - model_name: glm4.7 + litellm_params: + model: nvidia_nim/z-ai/glm4.7 + api_key: os.environ/NVIDIA_NIM_API_KEY + + - model_name: gemini-embedding-001 + litellm_params: + model: gemini/gemini-embedding-001 + api_key: os.environ/GEMINI_API_KEY + dimensions: 1536 + + - model_name: gemini-2.5-flash + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY + + - model_name: gemma-4-31b-it + litellm_params: + model: gemini/gemma-4-31b-it + api_key: os.environ/GEMINI_API_KEY + + - model_name: gemma-4-26b-a4b-it + litellm_params: + model: gemini/gemma-4-26b-a4b-it + api_key: os.environ/GEMINI_API_KEY + + # - model_name: gemini/* + # litellm_params: + # model: gemini/* + # api_key: os.environ/GEMINI_API_KEY + + - model_name: gemma4:31b-cloud + litellm_params: + model: ollama/gemma4:31b-cloud + api_base: https://ollama.com + api_key: os.environ/OLLAMA_API_KEY + + # - model_name: glm-4.7-flash + # litellm_params: + # model: cloudflare/@cf/zai-org/glm-4.7-flash + # api_key: os.environ/CLOUDFLARE_API_KEY + # account_id: os.environ/CLOUDFLARE_ACCOUNT_ID + + # - model_name: deepseek-v4-pro + # litellm_params: + # model: openai/deepseek-v4-pro + # api_base: https://opencode.ai/zen/go/v1 + # api_key: os.environ/OPENCODE_API_KEY + + # - model_name: qwen3.7-max + # litellm_params: + # model: anthropic/qwen3.7-max + # api_base: https://opencode.ai/zen/go/v1 + # api_key: os.environ/OPENCODE_API_KEY + +general_settings: + proxy_batch_write_at: 60 + store_model_in_db: true + store_prompts_in_spend_logs: false + +router_settings: + model_group_alias: + claude: openrouter-free + sonnet: openrouter-free + opus: openrouter-free + haiku: openrouter-free + opencode: openrouter-free + goclaw: openrouter-free + openclaw: openrouter-free + hermes: openrouter-free + redis_host: os.environ/REDIS_HOST + redis_port: os.environ/REDIS_PORT + redis_password: os.environ/REDIS_PASSWORD + enable_pre_call_check: true + +litellm_settings: + master_key: os.environ/LITELLM_MASTER_KEY + check_provider_endpoint: true + set_verbose: false + json_logs: true + log_raw_request_response: false + # turn_off_message_logging: false + # redact_user_api_key_info: false + service_callback: ["prometheus_system"] + drop_params: true + # max_budget: 100 + # budget_duration: 30d + num_retries: 1 + request_timeout: 120 + telemetry: false + cache: true + cache_params: + type: redis + host: os.environ/REDIS_HOST + port: os.environ/REDIS_PORT + password: os.environ/REDIS_PASSWORD + namespace: "litellm_cache" + ttl: 600 + success_callback: + # - "langfuse" + - "prometheus" + failure_callback: + # - "langfuse" + - "prometheus"