feat(litellm): wire SearXNG search, pgvector knowledgebase, and memory ingestion
- search_tools block in litellm-config.yaml (SearXNG as a first-class search_provider, standalone /v1/search endpoint, not a model tool) plus extra_hosts on the litellm service so it can resolve search.home. - New embedding-server (nomic-embed-text-v1.5 on a second llama.cpp instance), pgvector-db, and litellm-pgvector services — LiteLLM's native knowledgebase feature has no Qdrant backend, so this is the only self-hosted path (docs/research/litellm-knowledgebase.md). - vector_store_registry + local-embedding model entry in litellm-config.yaml, wiring it together. - scripts/ingest-memory.sh to load data/memory.md and data/claude-legacy-memory.md into the knowledgebase. - docs/memory-knowledgebase.md documenting the whole setup; data/ gitignored (personal memory content, not meant to be committed). - New .env vars (SEARXNG_LAN_IP, PGVECTOR_DB_PASSWORD, LITELLM_PGVECTOR_API_KEY, LITELLM_PGVECTOR_EMBEDDING_KEY, EMBEDDING_MODEL_FILE) and generate-secrets.sh support for the auto-generatable ones. Resolves #22 and #23 (wayfinder map #21). Not yet verified on real hardware — see #24. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -35,6 +35,36 @@ services:
|
||||
- "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
|
||||
- "lazytainer.group.llamaserver.minPacketThreshold=2"
|
||||
|
||||
embedding-server:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-rocm
|
||||
container_name: embedding-server
|
||||
devices:
|
||||
- /dev/kfd
|
||||
- /dev/dri
|
||||
group_add:
|
||||
- video
|
||||
- render
|
||||
security_opt:
|
||||
- seccomp=unconfined
|
||||
ipc: host
|
||||
volumes:
|
||||
- models:/models
|
||||
command: >
|
||||
-m /models/${EMBEDDING_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf}
|
||||
--host 0.0.0.0
|
||||
--port 8080
|
||||
--embeddings
|
||||
--pooling mean
|
||||
--n-gpu-layers 999
|
||||
--ctx-size 8192
|
||||
# A dedicated embedding model — the chat model isn't embedding-trained
|
||||
# and llama.cpp serves one model per process, so this is a second small
|
||||
# instance, not a mode switch on llama-server. See
|
||||
# docs/research/litellm-knowledgebase.md. Small enough (~150MB Q8) to
|
||||
# run alongside the chat model's ~19.6GB in the R9700's 32GB VRAM.
|
||||
restart: unless-stopped
|
||||
networks: [ai-stack]
|
||||
|
||||
# ponytail: one-off downloader, not a standing service — run via
|
||||
# `docker compose --profile tools run --rm downloader` (see scripts/download-model.sh).
|
||||
# Keeps the model file inside the named `models` volume instead of a host bind-mount.
|
||||
@@ -52,6 +82,20 @@ services:
|
||||
curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
|
||||
https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
|
||||
|
||||
# ponytail: same one-off pattern as `downloader`, for the embedding model —
|
||||
# run via `docker compose --profile tools run --rm downloader-embedding`.
|
||||
downloader-embedding:
|
||||
image: curlimages/curl:latest
|
||||
profiles: ["tools"]
|
||||
user: root
|
||||
volumes:
|
||||
- models:/models
|
||||
entrypoint: ["sh", "-c"]
|
||||
command:
|
||||
- >
|
||||
curl -L --fail --create-dirs -o /models/${EMBEDDING_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf}
|
||||
https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/${EMBEDDING_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf}
|
||||
|
||||
qdrant:
|
||||
image: qdrant/qdrant:latest
|
||||
container_name: qdrant
|
||||
@@ -107,6 +151,12 @@ services:
|
||||
env_file: .env
|
||||
environment:
|
||||
- DATABASE_URL=postgresql://litellm:${LITELLM_DB_PASSWORD}@litellm-db:5432/litellm
|
||||
# The litellm container only joins the ai-stack bridge network, which has
|
||||
# no visibility into the LAN's mDNS/local-DNS names — search.home won't
|
||||
# resolve without this. Set SEARXNG_LAN_IP in .env to its stable LAN IP
|
||||
# (static DHCP reservation recommended). See docs/research/litellm-searxng-search.md.
|
||||
extra_hosts:
|
||||
- "search.home:${SEARXNG_LAN_IP}"
|
||||
command: ["--config", "/app/config.yaml", "--port", "4000"]
|
||||
ports:
|
||||
# published for LAN access (proxy.ai.home) and, via NPM, proxy.ai.haylan.ch —
|
||||
@@ -141,6 +191,59 @@ services:
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
# Separate Postgres instance (with the pgvector extension) for the
|
||||
# knowledgebase — NOT the same database as litellm-db, which is plain
|
||||
# postgres:16-alpine and has no vector extension installed. See
|
||||
# docs/research/litellm-knowledgebase.md.
|
||||
pgvector-db:
|
||||
image: pgvector/pgvector:pg16
|
||||
container_name: pgvector-db
|
||||
env_file: .env
|
||||
environment:
|
||||
- POSTGRES_USER=litellm_pgvector
|
||||
- POSTGRES_PASSWORD=${PGVECTOR_DB_PASSWORD}
|
||||
- POSTGRES_DB=litellm_pgvector
|
||||
volumes:
|
||||
- pgvector-db-data:/var/lib/postgresql/data
|
||||
restart: unless-stopped
|
||||
networks: [ai-stack]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -d litellm_pgvector -U litellm_pgvector"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
# LiteLLM's native knowledgebase/vector-store feature has no Qdrant backend
|
||||
# (the qdrant service above only serves Open WebUI's own RAG/Memory) — this
|
||||
# companion service (github.com/BerriAI/litellm-pgvector) is the only
|
||||
# self-hosted path. No published image exists yet, so this builds straight
|
||||
# from the upstream repo. See docs/research/litellm-knowledgebase.md.
|
||||
# ponytail: unverified against real hardware — Prisma migration behavior on
|
||||
# first boot and the exact vector_store_registry field names for the
|
||||
# pg_vector provider need a live smoke test. See issue #24.
|
||||
litellm-pgvector:
|
||||
build:
|
||||
context: https://github.com/BerriAI/litellm-pgvector.git
|
||||
container_name: litellm-pgvector
|
||||
depends_on:
|
||||
pgvector-db:
|
||||
condition: service_healthy
|
||||
litellm:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
- DATABASE_URL=postgresql://litellm_pgvector:${PGVECTOR_DB_PASSWORD}@pgvector-db:5432/litellm_pgvector
|
||||
- SERVER_API_KEY=${LITELLM_PGVECTOR_API_KEY}
|
||||
# Calls back into litellm for embeddings, same pattern as any other
|
||||
# workload — see docs/proxy-key-onboarding.md for issuing this key.
|
||||
- EMBEDDING__MODEL=local-embedding
|
||||
- EMBEDDING__BASE_URL=http://litellm:4000
|
||||
- EMBEDDING__API_KEY=${LITELLM_PGVECTOR_EMBEDDING_KEY}
|
||||
- EMBEDDING__DIMENSIONS=768
|
||||
expose:
|
||||
- "8000"
|
||||
restart: unless-stopped
|
||||
networks: [ai-stack]
|
||||
|
||||
lazytainer:
|
||||
image: ghcr.io/vmorganp/lazytainer:master
|
||||
container_name: lazytainer
|
||||
@@ -159,3 +262,4 @@ volumes:
|
||||
qdrant-data:
|
||||
openwebui-data:
|
||||
litellm-db-data:
|
||||
pgvector-db-data:
|
||||
|
||||
Reference in New Issue
Block a user