From 0aefb36a48905e65815fed6cb06e311feb1f5ef9 Mon Sep 17 00:00:00 2001 From: Haylan Date: Tue, 25 Aug 2026 07:11:15 +0200 Subject: [PATCH] Author LiteLLM AI proxy service (resolves #14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds litellm + litellm-db to docker-compose.yml, litellm-config.yaml with custom shadow-cost pricing (Claude Sonnet 5 reference, per #11) and a priority-scheduling stub (per #16, needs real-hardware smoke test), and required LITELLM_MASTER_KEY/SALT_KEY/DB_PASSWORD env vars. Untested on real hardware — that's #17. Open WebUI/coding CLIs still talk to llama.cpp directly, migration is #15. Co-Authored-By: Claude Sonnet 5 --- .env.example | 9 +++++++++ README.md | 11 +++++++++++ docker-compose.yml | 33 +++++++++++++++++++++++++++++++++ litellm-config.yaml | 24 ++++++++++++++++++++++++ 4 files changed, 77 insertions(+) create mode 100644 litellm-config.yaml diff --git a/.env.example b/.env.example index bd4c0fe..e07498a 100644 --- a/.env.example +++ b/.env.example @@ -19,3 +19,12 @@ OPENAI_API_KEY=local # --- Lazytainer --- # Seconds of inactivity before llama-server is stopped. 900 = 15 min. LAZYTAINER_INACTIVE_TIMEOUT=900 + +# --- LiteLLM proxy (see docs/proxy-key-onboarding.md, docs/network-access.md) --- +LITELLM_PORT=4000 +# Required — generate real random values before first run, e.g. `openssl rand -hex 32`. +# LITELLM_SALT_KEY encrypts stored data; do not change it after the first run +# (existing encrypted data becomes unreadable if you do). +LITELLM_MASTER_KEY= +LITELLM_SALT_KEY= +LITELLM_DB_PASSWORD=litellm diff --git a/README.md b/README.md index 6daec14..87d0077 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,14 @@ docker compose up -d Pointing Claude Code CLI, Kimi CLI, or OpenCode CLI at the local endpoint: see [`docs/coding-cli-setup.md`](docs/coding-cli-setup.md). **Known risk**: Qwen3.8-27B's tool-calling reliability against llama.cpp's Anthropic shim is not yet verified (open upstream parser bugs against its model lineage) — see `docs/research/qwen3.8-27b-tool-calling.md`. + +## AI proxy (LiteLLM) + +An [AI gateway/proxy](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) fronts llama.cpp: per-workload virtual keys, usage tracking, and a shadow cost estimate ("what this would have cost on Claude Sonnet 5"). Before `docker compose up -d`, set `LITELLM_MASTER_KEY` and `LITELLM_SALT_KEY` in `.env` (see `.env.example`). + +- Proxy API: `http://:4000/v1` locally, or `proxy.ai.home` / `proxy.ai.haylan.ch` once routed through NPM — see [`docs/network-access.md`](docs/network-access.md). +- Admin UI (`/ui`, key/budget management): LAN-only — see `docs/network-access.md`. +- Issuing a key for a new workload: [`docs/proxy-key-onboarding.md`](docs/proxy-key-onboarding.md). +- Request priority across workloads: [`docs/proxy-request-priority.md`](docs/proxy-request-priority.md). + +**Not yet done**: Open WebUI and the coding CLIs still talk to llama.cpp directly, not through this proxy — that migration is [issue #15](https://git.arthurerlich.de/haylan/LLM-Server/issues/15). **Not yet verified**: this config hasn't been smoke-tested on real hardware (LiteLLM's priority scheduler in particular is beta — see `docs/proxy-request-priority.md`) — see [issue #17](https://git.arthurerlich.de/haylan/LLM-Server/issues/17). diff --git a/docker-compose.yml b/docker-compose.yml index aa9f3e1..48631e9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,38 @@ services: restart: unless-stopped networks: [ai-stack] + litellm: + image: ghcr.io/berriai/litellm:main-stable + container_name: litellm + depends_on: + - litellm-db + - llama-server + volumes: + - ./litellm-config.yaml:/app/config.yaml:ro + environment: + - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY:?set a real master key in .env — see docs/proxy-key-onboarding.md} + - LITELLM_SALT_KEY=${LITELLM_SALT_KEY:?set a real salt key in .env, do not change after first run} + - DATABASE_URL=postgresql://litellm:${LITELLM_DB_PASSWORD:-litellm}@litellm-db:5432/litellm + command: ["--config", "/app/config.yaml", "--port", "4000"] + ports: + # published for LAN access (proxy.ai.home) and, via NPM, proxy.ai.haylan.ch — + # NPM must deny the /ui path on the external host. See docs/network-access.md. + - "${LITELLM_PORT:-4000}:4000" + restart: unless-stopped + networks: [ai-stack] + + litellm-db: + image: postgres:16-alpine + container_name: litellm-db + environment: + - POSTGRES_USER=litellm + - POSTGRES_PASSWORD=${LITELLM_DB_PASSWORD:-litellm} + - POSTGRES_DB=litellm + volumes: + - litellm-db-data:/var/lib/postgresql/data + restart: unless-stopped + networks: [ai-stack] + lazytainer: image: ghcr.io/vmorganp/lazytainer:master container_name: lazytainer @@ -91,3 +123,4 @@ volumes: models: qdrant-data: openwebui-data: + litellm-db-data: diff --git a/litellm-config.yaml b/litellm-config.yaml new file mode 100644 index 0000000..93b5bb3 --- /dev/null +++ b/litellm-config.yaml @@ -0,0 +1,24 @@ +model_list: + - model_name: qwen3.8-27b-local + litellm_params: + model: openai/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} + api_base: http://llama-server:8080/v1 + api_key: local + model_info: + # Shadow cloud-cost estimate — priced against Claude Sonnet 5's published + # rate, not real spend (this proxy only ever routes to the local model). + # Source: https://platform.claude.com/docs/en/about-claude/pricing, + # checked 2026-08-25. Update these two numbers if that page changes. + input_cost_per_token: 0.000002 # $2 / MTok + output_cost_per_token: 0.00001 # $10 / MTok + +router_settings: + # ponytail: LiteLLM's request-prioritization scheduler is beta (see + # docs/proxy-request-priority.md) — exact settings key/shape must be + # confirmed against LiteLLM's current docs and smoke-tested against + # llama.cpp before workloads depend on it. Single-instance deployment, + # no Redis configured — add one only if the scheduler turns out to need it. + enable_priority_scheduling: true + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY