diff --git a/.env.example b/.env.example index b8ada63..5b3cffd 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,8 @@ -# Copy to .env and adjust. All values below are defaults baked into -# docker-compose.yml — only uncomment/change what you actually want to override. +# Copy to .env and adjust — or just run ./scripts/update.sh, which creates +# .env from this file and fills in every secret/key below it can generate +# itself (see each var's comment). All values below are defaults baked into +# docker-compose.yml — only uncomment/change what you actually want to +# override. # --- llama.cpp / model --- LLAMA_MODEL_FILE=Qwen3.8-27B-UD-Q4_K_XL.gguf @@ -11,8 +14,8 @@ LLAMA_CTX_SIZE=65536 # --- Open WebUI --- WEBUI_PORT=8008 -# Required — create an "openwebui" virtual key in LiteLLM's Admin UI first -# (see docs/proxy-key-onboarding.md), then paste it here. +# Minted automatically by ./scripts/update.sh — leave blank. Manual fallback: +# docs/proxy-key-onboarding.md. OPENWEBUI_LITELLM_KEY= # --- Lazytainer --- @@ -23,32 +26,33 @@ LAZYTAINER_INACTIVE_TIMEOUT=900 EMBEDDING_MODEL_FILE=nomic-embed-text-v1.5.Q8_0.gguf # --- SearXNG web search (see docs/research/litellm-searxng-search.md) --- -# Required — the litellm container can't resolve search.home's LAN mDNS -# name on its own; this becomes an extra_hosts entry. Use a static -# DHCP-reserved IP so it doesn't drift. +# Resolved automatically by ./scripts/update.sh from search.home on this +# host — leave blank. Only set by hand if that resolution fails (e.g. +# search.home isn't a static DHCP reservation and its IP drifted). SEARXNG_LAN_IP= # --- LiteLLM proxy (see docs/proxy-key-onboarding.md, docs/network-access.md) --- LITELLM_PORT=4000 -# Required — generate real random values before first run, e.g. `openssl rand -hex 32`. -# LITELLM_SALT_KEY encrypts stored data; do not change it after the first run -# (existing encrypted data becomes unreadable if you do). -# Required — generate a real random value before first run, e.g. `openssl rand -hex 32`. +# Random values, filled in automatically by ./scripts/update.sh — leave +# blank. LITELLM_SALT_KEY encrypts stored data; do not change it after the +# first run (existing encrypted data becomes unreadable if you do). LITELLM_MASTER_KEY= LITELLM_SALT_KEY= LITELLM_DB_PASSWORD= # Admin UI login (https:///ui). Without these, LiteLLM falls back to # username "admin" / password = LITELLM_MASTER_KEY — set these instead so the -# master key never has to be typed into the browser. +# master key never has to be typed into the browser. UI_PASSWORD is filled +# in automatically by ./scripts/update.sh if blank. UI_USERNAME=admin UI_PASSWORD= # --- Knowledgebase (pgvector + memory-retrieval, see docs/memory-knowledgebase.md) --- -# Required — random values, e.g. `openssl rand -hex 32`. +# Random value, filled in automatically by ./scripts/update.sh — leave blank. PGVECTOR_DB_PASSWORD= # Auth key memory-retrieval requires on its own API (its SERVER_API_KEY). +# Random value, filled in automatically by ./scripts/update.sh — leave blank. MEMORY_RETRIEVAL_API_KEY= # A virtual key memory-retrieval uses to call back into litellm for -# embeddings — create it in the Admin UI like any other workload key -# (see docs/proxy-key-onboarding.md), name it "memory-retrieval". +# embeddings. Minted automatically by ./scripts/update.sh — leave blank. +# Manual fallback: docs/proxy-key-onboarding.md. MEMORY_RETRIEVAL_EMBEDDING_KEY= diff --git a/README.md b/README.md index ba2660b..1ea3ef4 100644 --- a/README.md +++ b/README.md @@ -7,30 +7,22 @@ See the wayfinder map ([issue #1](https://git.arthurerlich.de/haylan/LLM-Server/ ## Quickstart ```bash -cp .env.example .env -# set LITELLM_MASTER_KEY / LITELLM_SALT_KEY (openssl rand -hex 32), see .env.example ./scripts/download-model.sh -docker compose up -d litellm litellm-db llama-server qdrant # bring the proxy up first +./scripts/update.sh ``` -Log into LiteLLM's Admin UI (`http://:4000/ui`), create an `openwebui` virtual key (see [`docs/proxy-key-onboarding.md`](docs/proxy-key-onboarding.md)), set `OPENWEBUI_LITELLM_KEY` in `.env` to it, then: - -```bash -docker compose up -d -``` +`update.sh` creates `.env` from `.env.example` if missing, fills in every secret and per-workload virtual key it can generate itself (random secrets via `openssl`, `OPENWEBUI_LITELLM_KEY`/`MEMORY_RETRIEVAL_EMBEDDING_KEY` minted through LiteLLM's own `/key/generate` API, `SEARXNG_LAN_IP` resolved from `search.home` on this host), then pulls/builds/brings up the whole stack. Safe to re-run any time — it only fills in what's still blank and only recreates what changed. See [`docs/proxy-key-onboarding.md`](docs/proxy-key-onboarding.md) if a key mint fails and needs doing by hand. - Open WebUI: `http://:3000` locally, or `ai.home` / `ai.haylan.ch` once routed through Nginx Proxy Manager — see [`docs/network-access.md`](docs/network-access.md). First signup becomes the admin account (`WEBUI_AUTH` is on). - llama.cpp's own API is internal-only now — everything routes through the AI proxy below. Pointing Claude Code CLI, Kimi CLI, or OpenCode CLI at the local endpoint: see [`docs/coding-cli-setup.md`](docs/coding-cli-setup.md). -**Applying updates**: after pulling changes (or editing `docker-compose.yml`/`litellm-config.yaml`/`.env` yourself), run `./scripts/update.sh` — pulls, validates the compose config, rebuilds/re-pulls images, and recreates whatever changed. Safe to run any time. - **Known risk**: Qwen3.8-27B's tool-calling reliability against llama.cpp's Anthropic shim is not yet verified (open upstream parser bugs against its model lineage) — see `docs/research/qwen3.8-27b-tool-calling.md`. ## AI proxy (LiteLLM) -An [AI gateway/proxy](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) fronts llama.cpp: per-workload virtual keys, usage tracking, and a shadow cost estimate ("what this would have cost on Claude Sonnet 5"). Before `docker compose up -d`, set `LITELLM_MASTER_KEY` and `LITELLM_SALT_KEY` in `.env` (see `.env.example`). +An [AI gateway/proxy](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) fronts llama.cpp: per-workload virtual keys, usage tracking, and a shadow cost estimate ("what this would have cost on Claude Sonnet 5"). `./scripts/update.sh` handles `LITELLM_MASTER_KEY`/`LITELLM_SALT_KEY` and every other secret (see `.env.example`). - Proxy API: `http://:4000/v1` locally, or `proxy.ai.home` / `proxy.ai.haylan.ch` once routed through NPM — see [`docs/network-access.md`](docs/network-access.md). - Admin UI (`/ui`, key/budget management): LAN-only — see `docs/network-access.md`. diff --git a/docs/memory-knowledgebase.md b/docs/memory-knowledgebase.md index c1cc90e..d4c08c3 100644 --- a/docs/memory-knowledgebase.md +++ b/docs/memory-knowledgebase.md @@ -17,7 +17,7 @@ curl http://:4000/v1/search/searxng-search \ Because this doesn't ask the model to emit a tool call, it sidesteps Qwen3.8-27B's known-flaky tool-calling (`docs/research/qwen3.8-27b-tool-calling.md`) entirely. Open WebUI's own web-search setting can point at this endpoint the same way. -Requires `SEARXNG_LAN_IP` set in `.env` (SearXNG's stable LAN IP — use a static DHCP reservation) so the `litellm` container can resolve `search.home` via `extra_hosts`. Full research: `docs/research/litellm-searxng-search.md`. +Requires `SEARXNG_LAN_IP` set in `.env` so the `litellm` container can resolve `search.home` via `extra_hosts` — `./scripts/update.sh` resolves and fills this in automatically from the host's own DNS if it's blank (use a static DHCP reservation for `search.home` so it doesn't drift). Full research: `docs/research/litellm-searxng-search.md`. ## Knowledgebase (vector store / RAG) @@ -37,7 +37,7 @@ docker compose --profile tools run --rm downloader-embedding # fetch the embed docker compose up -d embedding-server pgvector-db memory-retrieval ``` -Create a `memory-retrieval` virtual key in LiteLLM's Admin UI (per `docs/proxy-key-onboarding.md`) and set it as `MEMORY_RETRIEVAL_EMBEDDING_KEY` in `.env` — it calls back into `litellm` for embeddings, same as any other workload. +`./scripts/update.sh` mints `MEMORY_RETRIEVAL_EMBEDDING_KEY` automatically (a `memory-retrieval` virtual key via LiteLLM's own API) if it's blank — it calls back into `litellm` for embeddings, same as any other workload. See `docs/proxy-key-onboarding.md` if a mint fails and it needs doing by hand. ### Loading memory into it diff --git a/docs/proxy-key-onboarding.md b/docs/proxy-key-onboarding.md index 205c52d..697c1b9 100644 --- a/docs/proxy-key-onboarding.md +++ b/docs/proxy-key-onboarding.md @@ -2,6 +2,8 @@ How to issue a new per-workload API key against the LiteLLM proxy (see [issue #10](https://git.arthurerlich.de/haylan/LLM-Server/issues/10) / `docs/research/proxy-tool-choice.md`), so a new workload (a code-reviewer tool, Paperless-OCR, Gitea code review, etc.) gets its own key and its own visible usage/spend. +`OPENWEBUI_LITELLM_KEY` and `MEMORY_RETRIEVAL_EMBEDDING_KEY` — the two keys this stack's own services need — are minted automatically by `./scripts/update.sh` via the same API `curl` shows below; the steps here are for any other workload, or for those two if the automatic mint ever fails. + ## Create the key 1. Log into LiteLLM's Admin UI (`/ui` on the proxy's deployed URL). @@ -9,6 +11,16 @@ How to issue a new per-workload API key against the LiteLLM proxy (see [issue #1 3. Name it `-` — a short slug matching the workload, e.g. `paperless-ocr`, `gitea-code-review`, `openwebui`. This name is the ledger: LiteLLM's dashboard lists keys by name, so there's no separate tracking doc to keep in sync — name it clearly and the Usage tab tells you the rest (spend, last used, etc.). 4. Leave budget and rate limits unset (unlimited) by default. This is a shadow-cost estimate for fun, not real accounting or resource protection — see `docs/research/proxy-shadow-pricing.md`. Only set a budget if a specific workload turns out to need a tripwire. +Or the same thing over the API (what `update.sh` does): + +```bash +curl -sf -X POST "http://:4000/key/generate" \ + -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \ + -H "Content-Type: application/json" \ + -d '{"key_alias": "-"}' +# -> {"key": "sk-...", ...} +``` + ## Hand it to the workload Drop the key into that workload's own `.env` (or equivalent config) — never into this repo. Each workload's config is the source of truth for its own credential, same pattern as `.env.example` in this repo for the existing stack. diff --git a/scripts/generate-secrets.sh b/scripts/generate-secrets.sh deleted file mode 100755 index ce8339e..0000000 --- a/scripts/generate-secrets.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Generates random values for the secrets docker-compose.yml requires -# (LITELLM_MASTER_KEY, LITELLM_SALT_KEY, LITELLM_DB_PASSWORD, UI_PASSWORD, -# PGVECTOR_DB_PASSWORD, MEMORY_RETRIEVAL_API_KEY) and writes them into .env — -# creating it from .env.example first if it doesn't exist. -# -# ponytail: only fills in blank values, never overwrites ones you've already -# set — safe to re-run. Re-running won't touch LITELLM_SALT_KEY once it's -# set; changing it after first run makes existing encrypted data unreadable. -set -euo pipefail -cd "$(dirname "$0")/.." - -[ -f .env ] || cp .env.example .env - -set_if_blank() { - local key="$1" value="$2" - if grep -qE "^${key}=.*[^[:space:]]" .env; then - echo "${key}: already set, skipping." - else - sed -i "s|^${key}=.*|${key}=${value}|" .env - echo "${key}: generated." - fi -} - -set_if_blank LITELLM_MASTER_KEY "$(openssl rand -hex 32)" -set_if_blank LITELLM_SALT_KEY "$(openssl rand -hex 32)" -set_if_blank LITELLM_DB_PASSWORD "$(openssl rand -hex 32)" -set_if_blank UI_PASSWORD "$(openssl rand -hex 16)" -set_if_blank PGVECTOR_DB_PASSWORD "$(openssl rand -hex 32)" -set_if_blank MEMORY_RETRIEVAL_API_KEY "$(openssl rand -hex 32)" - -echo "Done. Review .env, then set OPENWEBUI_LITELLM_KEY, MEMORY_RETRIEVAL_EMBEDDING_KEY, and SEARXNG_LAN_IP per docs/proxy-key-onboarding.md and docs/memory-knowledgebase.md." diff --git a/scripts/update.sh b/scripts/update.sh index 4da1225..f18324a 100644 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # The one command to run after any change to this repo (compose file, # litellm-config.yaml, .env, or a git pull) to bring the running stack in -# sync. Pulls, validates, rebuilds/re-pulls images, and recreates only what -# changed — safe to run any time, including with nothing to do. +# sync. Ensures secrets/keys exist, pulls, validates, rebuilds/re-pulls +# images, and recreates only what changed — safe to run any time, including +# with nothing to do. # # ponytail: no rollback/backup logic — this is a single-user homelab box, # not a fleet. If a bad config lands, `git revert` + re-run is the recovery @@ -10,6 +11,39 @@ set -euo pipefail cd "$(dirname "$0")/.." +[ -f .env ] || cp .env.example .env + +set_if_blank() { + local key="$1" value="$2" + if grep -qE "^${key}=.*[^[:space:]]" .env; then + echo "${key}: already set, skipping." + else + sed -i "s|^${key}=.*|${key}=${value}|" .env + echo "${key}: set." + fi +} + +echo "==> filling in missing secrets" +# Random values — safe to re-run, never overwrites what's already set. +# LITELLM_SALT_KEY especially: never change it after first run, existing +# encrypted data becomes unreadable if you do. +set_if_blank LITELLM_MASTER_KEY "$(openssl rand -hex 32)" +set_if_blank LITELLM_SALT_KEY "$(openssl rand -hex 32)" +set_if_blank LITELLM_DB_PASSWORD "$(openssl rand -hex 32)" +set_if_blank UI_PASSWORD "$(openssl rand -hex 16)" +set_if_blank PGVECTOR_DB_PASSWORD "$(openssl rand -hex 32)" +set_if_blank MEMORY_RETRIEVAL_API_KEY "$(openssl rand -hex 32)" + +echo "==> resolving SEARXNG_LAN_IP" +# search.home is a LAN mDNS/local-DNS name — resolvable from this host, just +# not from inside the litellm container (see docs/research/litellm-searxng-search.md). +searxng_ip="$(getent hosts search.home 2>/dev/null | awk '{print $1}' | head -1)" +if [ -n "$searxng_ip" ]; then + set_if_blank SEARXNG_LAN_IP "$searxng_ip" +else + echo "SEARXNG_LAN_IP: couldn't resolve search.home from this host, set it manually if still blank." +fi + echo "==> git pull" git pull --ff-only @@ -19,9 +53,40 @@ docker compose config -q echo "==> pulling images" docker compose pull --ignore-buildable -echo "==> rebuilding local-build services (e.g. litellm-pgvector)" +echo "==> rebuilding local-build services" docker compose build --pull +echo "==> bringing up litellm (needed to mint virtual keys below)" +docker compose up -d --wait litellm-db litellm + +# OPENWEBUI_LITELLM_KEY / MEMORY_RETRIEVAL_EMBEDDING_KEY are per-workload +# virtual keys, not random secrets — minted via LiteLLM's own API +# (docs/proxy-key-onboarding.md documents the manual Admin UI route; this is +# the same thing over the REST endpoint LITELLM_MASTER_KEY already +# authenticates against). +set -a && . ./.env && set +a +mint_key_if_blank() { + local key="$1" alias="$2" + if grep -qE "^${key}=.*[^[:space:]]" .env; then + echo "${key}: already set, skipping." + return + fi + local minted + minted=$(curl -sf -X POST "http://localhost:${LITELLM_PORT:-4000}/key/generate" \ + -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \ + -H "Content-Type: application/json" \ + -d "{\"key_alias\": \"${alias}\"}" | jq -r '.key') + if [ -n "$minted" ] && [ "$minted" != "null" ]; then + sed -i "s|^${key}=.*|${key}=${minted}|" .env + echo "${key}: minted." + else + echo "${key}: mint failed, create it by hand per docs/proxy-key-onboarding.md." + fi +} +mint_key_if_blank OPENWEBUI_LITELLM_KEY openwebui +mint_key_if_blank MEMORY_RETRIEVAL_EMBEDDING_KEY memory-retrieval +set -a && . ./.env && set +a + echo "==> recreating changed services" docker compose up -d --remove-orphans