diff --git a/.env.example b/.env.example index 3b81b0c..09698bc 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,15 @@ OPENWEBUI_LITELLM_KEY= # Seconds of inactivity before llama-server is stopped. 900 = 15 min. LAZYTAINER_INACTIVE_TIMEOUT=900 +# --- Embedding model (knowledgebase, see docs/memory-knowledgebase.md) --- +EMBEDDING_MODEL_FILE=nomic-embed-text-v1.5.Q8_0.gguf + +# --- SearXNG web search (see docs/research/litellm-searxng-search.md) --- +# Required — the litellm container can't resolve search.home's LAN mDNS +# name on its own; this becomes an extra_hosts entry. Use a static +# DHCP-reserved IP so it doesn't drift. +SEARXNG_LAN_IP= + # --- LiteLLM proxy (see docs/proxy-key-onboarding.md, docs/network-access.md) --- LITELLM_PORT=4000 # Required — generate real random values before first run, e.g. `openssl rand -hex 32`. @@ -33,3 +42,13 @@ LITELLM_DB_PASSWORD= # master key never has to be typed into the browser. UI_USERNAME=admin UI_PASSWORD= + +# --- Knowledgebase (pgvector, see docs/memory-knowledgebase.md) --- +# Required — random values, e.g. `openssl rand -hex 32`. +PGVECTOR_DB_PASSWORD= +# Auth key litellm-pgvector requires on its own API (its SERVER_API_KEY). +LITELLM_PGVECTOR_API_KEY= +# A virtual key litellm-pgvector uses to call back into litellm for +# embeddings — create it in the Admin UI like any other workload key +# (see docs/proxy-key-onboarding.md), name it "litellm-pgvector". +LITELLM_PGVECTOR_EMBEDDING_KEY= diff --git a/.gitignore b/.gitignore index a7afddf..74c31ad 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ .claude/ -.env \ No newline at end of file +.env +# Personal memory content ingested by scripts/ingest-memory.sh — not meant +# to be committed to this repo. +data/ +.leankg/ diff --git a/README.md b/README.md index 4c20620..37805a5 100644 --- a/README.md +++ b/README.md @@ -36,3 +36,7 @@ An [AI gateway/proxy](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) fr - Request priority across workloads: [`docs/proxy-request-priority.md`](docs/proxy-request-priority.md). Open WebUI and the coding CLIs (see [`docs/coding-cli-setup.md`](docs/coding-cli-setup.md)) route through the proxy now — llama-server has no published host port anymore. **Not yet verified**: none of this has been smoke-tested on real hardware (LiteLLM's priority scheduler in particular is beta — see `docs/proxy-request-priority.md`) — see [issue #17](https://git.arthurerlich.de/haylan/LLM-Server/issues/17). + +### Web search, knowledgebase, and memory + +The gateway also fronts SearXNG-backed web search and a pgvector-backed knowledgebase (loaded with `data/memory.md` / `data/claude-legacy-memory.md`), wired at the LiteLLM layer so every client gets them, not just Open WebUI — see [`docs/memory-knowledgebase.md`](docs/memory-knowledgebase.md). **Not yet verified on real hardware** — see [issue #24](https://git.arthurerlich.de/haylan/LLM-Server/issues/24). diff --git a/docker-compose.yml b/docker-compose.yml index d45cc70..3806a1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,36 @@ services: - "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}" - "lazytainer.group.llamaserver.minPacketThreshold=2" + embedding-server: + image: ghcr.io/ggml-org/llama.cpp:server-rocm + container_name: embedding-server + devices: + - /dev/kfd + - /dev/dri + group_add: + - video + - render + security_opt: + - seccomp=unconfined + ipc: host + volumes: + - models:/models + command: > + -m /models/${EMBEDDING_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf} + --host 0.0.0.0 + --port 8080 + --embeddings + --pooling mean + --n-gpu-layers 999 + --ctx-size 8192 + # A dedicated embedding model — the chat model isn't embedding-trained + # and llama.cpp serves one model per process, so this is a second small + # instance, not a mode switch on llama-server. See + # docs/research/litellm-knowledgebase.md. Small enough (~150MB Q8) to + # run alongside the chat model's ~19.6GB in the R9700's 32GB VRAM. + restart: unless-stopped + networks: [ai-stack] + # ponytail: one-off downloader, not a standing service — run via # `docker compose --profile tools run --rm downloader` (see scripts/download-model.sh). # Keeps the model file inside the named `models` volume instead of a host bind-mount. @@ -52,6 +82,20 @@ services: curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} + # ponytail: same one-off pattern as `downloader`, for the embedding model — + # run via `docker compose --profile tools run --rm downloader-embedding`. + downloader-embedding: + image: curlimages/curl:latest + profiles: ["tools"] + user: root + volumes: + - models:/models + entrypoint: ["sh", "-c"] + command: + - > + curl -L --fail --create-dirs -o /models/${EMBEDDING_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf} + https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/${EMBEDDING_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf} + qdrant: image: qdrant/qdrant:latest container_name: qdrant @@ -107,6 +151,12 @@ services: env_file: .env environment: - DATABASE_URL=postgresql://litellm:${LITELLM_DB_PASSWORD}@litellm-db:5432/litellm + # The litellm container only joins the ai-stack bridge network, which has + # no visibility into the LAN's mDNS/local-DNS names — search.home won't + # resolve without this. Set SEARXNG_LAN_IP in .env to its stable LAN IP + # (static DHCP reservation recommended). See docs/research/litellm-searxng-search.md. + extra_hosts: + - "search.home:${SEARXNG_LAN_IP}" command: ["--config", "/app/config.yaml", "--port", "4000"] ports: # published for LAN access (proxy.ai.home) and, via NPM, proxy.ai.haylan.ch — @@ -141,6 +191,59 @@ services: timeout: 5s retries: 10 + # Separate Postgres instance (with the pgvector extension) for the + # knowledgebase — NOT the same database as litellm-db, which is plain + # postgres:16-alpine and has no vector extension installed. See + # docs/research/litellm-knowledgebase.md. + pgvector-db: + image: pgvector/pgvector:pg16 + container_name: pgvector-db + env_file: .env + environment: + - POSTGRES_USER=litellm_pgvector + - POSTGRES_PASSWORD=${PGVECTOR_DB_PASSWORD} + - POSTGRES_DB=litellm_pgvector + volumes: + - pgvector-db-data:/var/lib/postgresql/data + restart: unless-stopped + networks: [ai-stack] + healthcheck: + test: ["CMD-SHELL", "pg_isready -d litellm_pgvector -U litellm_pgvector"] + interval: 5s + timeout: 5s + retries: 10 + + # LiteLLM's native knowledgebase/vector-store feature has no Qdrant backend + # (the qdrant service above only serves Open WebUI's own RAG/Memory) — this + # companion service (github.com/BerriAI/litellm-pgvector) is the only + # self-hosted path. No published image exists yet, so this builds straight + # from the upstream repo. See docs/research/litellm-knowledgebase.md. + # ponytail: unverified against real hardware — Prisma migration behavior on + # first boot and the exact vector_store_registry field names for the + # pg_vector provider need a live smoke test. See issue #24. + litellm-pgvector: + build: + context: https://github.com/BerriAI/litellm-pgvector.git + container_name: litellm-pgvector + depends_on: + pgvector-db: + condition: service_healthy + litellm: + condition: service_healthy + environment: + - DATABASE_URL=postgresql://litellm_pgvector:${PGVECTOR_DB_PASSWORD}@pgvector-db:5432/litellm_pgvector + - SERVER_API_KEY=${LITELLM_PGVECTOR_API_KEY} + # Calls back into litellm for embeddings, same pattern as any other + # workload — see docs/proxy-key-onboarding.md for issuing this key. + - EMBEDDING__MODEL=local-embedding + - EMBEDDING__BASE_URL=http://litellm:4000 + - EMBEDDING__API_KEY=${LITELLM_PGVECTOR_EMBEDDING_KEY} + - EMBEDDING__DIMENSIONS=768 + expose: + - "8000" + restart: unless-stopped + networks: [ai-stack] + lazytainer: image: ghcr.io/vmorganp/lazytainer:master container_name: lazytainer @@ -159,3 +262,4 @@ volumes: qdrant-data: openwebui-data: litellm-db-data: + pgvector-db-data: diff --git a/docs/memory-knowledgebase.md b/docs/memory-knowledgebase.md new file mode 100644 index 0000000..cc15581 --- /dev/null +++ b/docs/memory-knowledgebase.md @@ -0,0 +1,64 @@ +# Knowledgebase, memory, and web search + +Three gateway-level capabilities added on top of the [AI gateway/proxy](https://git.arthurerlich.de/haylan/LLM-Server/issues/9), so every client behind LiteLLM gets them — not just Open WebUI. See [issue #21](https://git.arthurerlich.de/haylan/LLM-Server/issues/21) for the rationale. + +**Not yet verified on real hardware** — see [issue #24](https://git.arthurerlich.de/haylan/LLM-Server/issues/24). In particular: `litellm-pgvector`'s Prisma migrations on first boot, and the exact `vector_store_registry` field names for the `pg_vector` provider. + +## Web search (SearXNG) + +`litellm-config.yaml`'s `search_tools` block wires the LAN's SearXNG instance in as a **standalone REST endpoint**, not a model-callable tool — call it directly: + +```bash +curl http://:4000/v1/search/searxng-search \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"query": "...", "max_results": 5}' +``` + +Because this doesn't ask the model to emit a tool call, it sidesteps Qwen3.8-27B's known-flaky tool-calling (`docs/research/qwen3.8-27b-tool-calling.md`) entirely. Open WebUI's own web-search setting can point at this endpoint the same way. + +Requires `SEARXNG_LAN_IP` set in `.env` (SearXNG's stable LAN IP — use a static DHCP reservation) so the `litellm` container can resolve `search.home` via `extra_hosts`. Full research: `docs/research/litellm-searxng-search.md`. + +## Knowledgebase (vector store / RAG) + +LiteLLM's native knowledgebase feature has **no Qdrant backend** — the `qdrant` service in this stack only serves Open WebUI's own separate RAG/Memory feature and is unrelated to this. The only self-hosted path is [litellm-pgvector](https://github.com/BerriAI/litellm-pgvector), a companion service backed by its own Postgres+pgvector database (`pgvector-db`), which this stack now runs alongside `litellm`. Full research: `docs/research/litellm-knowledgebase.md`. + +New pieces: + +- **`embedding-server`** — a second llama.cpp instance (small footprint, `nomic-embed-text-v1.5`) serving `/v1/embeddings`. The chat model isn't embedding-trained and llama.cpp serves one model per process, so this can't just be a flag on `llama-server`. +- **`pgvector-db`** — Postgres with the pgvector extension, separate from `litellm-db`. +- **`litellm-pgvector`** — the connector service; built straight from its upstream repo (no published image exists). +- `litellm-config.yaml`'s `local-embedding` model entry and `vector_store_registry` block, tying it together. + +### First-time setup + +```bash +docker compose --profile tools run --rm downloader-embedding # fetch the embedding model +docker compose up -d embedding-server pgvector-db litellm-pgvector +``` + +Create a `litellm-pgvector` virtual key in LiteLLM's Admin UI (per `docs/proxy-key-onboarding.md`) and set it as `LITELLM_PGVECTOR_EMBEDDING_KEY` in `.env` — the connector calls back into `litellm` for embeddings, same as any other workload. + +### Loading memory into it + +`data/memory.md` and `data/claude-legacy-memory.md` — Claude-memory-style fact files — get loaded via: + +```bash +./scripts/ingest-memory.sh +``` + +One chunk per fact/paragraph line, tagged with `source`/`section` metadata. Re-run after editing either file (see the script's header comment for the no-dedup caveat). + +### Querying it + +Via the OpenAI Assistants-style `file_search` tool on a chat completion: + +```json +{ + "model": "qwen3.8-27b-local", + "messages": [...], + "tools": [{"type": "file_search", "vector_store_ids": ["memory-and-notes"]}] +} +``` + +or directly: `POST /v1/vector_stores/memory-and-notes/search` with `{"query": "..."}`. diff --git a/litellm-config.yaml b/litellm-config.yaml index 979fac3..9639172 100644 --- a/litellm-config.yaml +++ b/litellm-config.yaml @@ -22,6 +22,44 @@ model_list: input_cost_per_token: 0.000002 # $2 / MTok output_cost_per_token: 0.00001 # $10 / MTok + - model_name: local-embedding + litellm_params: + # Served by the dedicated embedding-server (nomic-embed-text-v1.5), not + # the chat model — see docker-compose.yml. Called by litellm-pgvector + # to embed knowledgebase content, and available directly at + # /v1/embeddings for anything else that wants it. + model: openai/local-embedding + api_base: http://embedding-server:8080/v1 + api_key: local + model_info: + mode: embedding + +# SearXNG-backed web search — a standalone REST endpoint (/v1/search/searxng-search), +# NOT a model-callable tool and not auto-injected into chat completions. See +# docs/research/litellm-searxng-search.md. Requires the litellm container to +# resolve search.home — see the `extra_hosts` entry in docker-compose.yml. +search_tools: + - search_tool_name: searxng-search + litellm_params: + search_provider: searxng + api_base: http://search.home/ + +# Knowledgebase / RAG, backed by the litellm-pgvector companion service (NOT +# Qdrant — LiteLLM's native vector-store feature has no Qdrant provider, see +# docs/research/litellm-knowledgebase.md). vector_store_id is this proxy's +# own identifier for the store, not assigned by a backend. +# ponytail: field names here (custom_llm_provider: pg_vector, api_base +# pointed at litellm-pgvector) are the best fit from the litellm-pgvector +# README, not confirmed against a running deploy yet — smoke-test before +# relying on it. See issue #24. +vector_store_registry: + - vector_store_name: memory-and-notes + litellm_params: + vector_store_id: "memory-and-notes" + custom_llm_provider: pg_vector + api_base: http://litellm-pgvector:8000 + embedding_model: local-embedding + router_settings: # ponytail: LiteLLM's request-prioritization scheduler is beta (see # docs/proxy-request-priority.md) — exact settings key/shape must be diff --git a/scripts/generate-secrets.sh b/scripts/generate-secrets.sh index 23024cf..2368c63 100755 --- a/scripts/generate-secrets.sh +++ b/scripts/generate-secrets.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Generates random values for the secrets docker-compose.yml requires -# (LITELLM_MASTER_KEY, LITELLM_SALT_KEY, LITELLM_DB_PASSWORD, UI_PASSWORD) -# and writes them into .env — creating it from .env.example first if it -# doesn't exist. +# (LITELLM_MASTER_KEY, LITELLM_SALT_KEY, LITELLM_DB_PASSWORD, UI_PASSWORD, +# PGVECTOR_DB_PASSWORD, LITELLM_PGVECTOR_API_KEY) and writes them into .env — +# creating it from .env.example first if it doesn't exist. # # ponytail: only fills in blank values, never overwrites ones you've already # set — safe to re-run. Re-running won't touch LITELLM_SALT_KEY once it's @@ -26,5 +26,7 @@ set_if_blank LITELLM_MASTER_KEY "$(openssl rand -hex 32)" set_if_blank LITELLM_SALT_KEY "$(openssl rand -hex 32)" set_if_blank LITELLM_DB_PASSWORD "$(openssl rand -hex 32)" set_if_blank UI_PASSWORD "$(openssl rand -hex 16)" +set_if_blank PGVECTOR_DB_PASSWORD "$(openssl rand -hex 32)" +set_if_blank LITELLM_PGVECTOR_API_KEY "$(openssl rand -hex 32)" -echo "Done. Review .env, then set OPENWEBUI_LITELLM_KEY per docs/proxy-key-onboarding.md." +echo "Done. Review .env, then set OPENWEBUI_LITELLM_KEY, LITELLM_PGVECTOR_EMBEDDING_KEY, and SEARXNG_LAN_IP per docs/proxy-key-onboarding.md and docs/memory-knowledgebase.md." diff --git a/scripts/ingest-memory.sh b/scripts/ingest-memory.sh new file mode 100644 index 0000000..5b62808 --- /dev/null +++ b/scripts/ingest-memory.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Loads data/memory.md and data/claude-legacy-memory.md into the LiteLLM +# knowledgebase (the "memory-and-notes" vector store, see litellm-config.yaml) +# via litellm-pgvector's batch-embeddings endpoint. +# +# ponytail: one chunk per non-empty, non-heading line — both source files are +# already one fact/paragraph per line (no hard-wrapping), so this needs no +# real chunking logic. Re-run after editing either file; there's no dedup, so +# this appends duplicates on a second run against unchanged content — clear +# the store first (DELETE the vector_store_id's rows) if you need a clean +# reload. +set -euo pipefail +cd "$(dirname "$0")/.." + +[ -f .env ] && set -a && . ./.env && set +a + +: "${LITELLM_PGVECTOR_API_KEY:?Set LITELLM_PGVECTOR_API_KEY in .env first}" +LITELLM_PGVECTOR_URL="${LITELLM_PGVECTOR_URL:-http://localhost:8000}" +VECTOR_STORE_ID="memory-and-notes" + +# Must match litellm-config.yaml's vector_store_registry entry — the +# registry just points at a store the backend must already know about. +# Ignores failure if it already exists (no documented idempotency check). +curl -sf -X POST "${LITELLM_PGVECTOR_URL}/v1/vector_stores" \ + -H "Authorization: Bearer ${LITELLM_PGVECTOR_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "{\"name\": \"${VECTOR_STORE_ID}\"}" > /dev/null 2>&1 || true + +ingest_file() { + local file="$1" section="" + local batch="[]" + while IFS= read -r line; do + case "$line" in + "#"*) section="${line#\# }"; section="${section#\#\# }"; continue ;; + ""|"---") continue ;; + esac + batch=$(jq --arg content "$line" --arg source "$file" --arg section "$section" \ + '. += [{"content": $content, "metadata": {"source": $source, "section": $section}}]' <<<"$batch") + done < "$file" + + echo "Ingesting $(jq 'length' <<<"$batch") chunks from $file..." + curl -sf -X POST "${LITELLM_PGVECTOR_URL}/v1/vector_stores/${VECTOR_STORE_ID}/embeddings/batch" \ + -H "Authorization: Bearer ${LITELLM_PGVECTOR_API_KEY}" \ + -H "Content-Type: application/json" \ + -d "$batch" > /dev/null +} + +ingest_file data/memory.md +ingest_file data/claude-legacy-memory.md +echo "Done."