diff --git a/.env.example b/.env.example index 95d1335..7f45eb0 100644 --- a/.env.example +++ b/.env.example @@ -28,6 +28,12 @@ LLAMA_GPU_LAYERS=999 # of dropping context: --cache-type-k/v q8_0 roughly halves it. LLAMA_CTX_SIZE=131072 +# --- embedding-server (OmniRoute memory) --- +# bge-small-en-v1.5, CPU-only, 384-dim output — see docker-compose.yml's +# embedding-server service comment for why this is a second model/process +# rather than reusing llama-server. +EMBEDDING_MODEL_FILE=bge-small-en-v1.5-q8_0.gguf + # --- Lazytainer --- # Seconds of inactivity before llama-server is stopped. 900 = 15 min. LAZYTAINER_INACTIVE_TIMEOUT=900 diff --git a/README.md b/README.md index 40bdab7..3153876 100644 --- a/README.md +++ b/README.md @@ -37,4 +37,4 @@ The gateway also fronts SearXNG-backed web search — see `docs/research/litellm ## What's not here - **Open WebUI** — this stack has no chat UI; every client is a coding CLI. Removed rather than kept idle. -- **Gateway-level knowledgebase/memory** (`litellm-pgvector`, `pgvector-db`, a dedicated embedding model) — removed as unwanted, unrelated to OmniRoute's own lack of parity with it (see issue #31's #34). Superseded by OmniRoute's own memory feature, backed by `qdrant` (bare service — wired up as a memory provider by hand in the dashboard, no static config here). +- **Gateway-level knowledgebase/memory** (`litellm-pgvector`, `pgvector-db`, a dedicated embedding model) — removed as unwanted, unrelated to OmniRoute's own lack of parity with it (see issue #31's #34). Superseded by OmniRoute's own memory feature, backed by `qdrant` and `embedding-server` (bge-small-en-v1.5, CPU-only) — both bare services, wired up as a memory provider by hand in the dashboard, no static routing config here. diff --git a/docker-compose.yml b/docker-compose.yml index f1de11d..4199b09 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,7 +39,7 @@ services: # `docker compose --profile tools run --rm downloader`. Folded into # scripts/update.sh, which runs this every time; the `test -f` guard is # what makes that safe to re-run without re-downloading. Keeps the model - # file inside the named `models` volume instead of a host bind-mount. + # files inside the named `models` volume instead of a host bind-mount. downloader: image: curlimages/curl:latest profiles: ["tools"] @@ -54,7 +54,34 @@ services: test -f /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} && echo "already downloaded, skipping" || curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} - https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} + https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}; + test -f /models/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf} && + echo "already downloaded, skipping" || + curl -L --fail --create-dirs -o /models/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf} + https://huggingface.co/ggml-org/bge-small-en-v1.5-Q8_0-GGUF/resolve/main/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf} + + # Dedicated embedding model for OmniRoute's memory feature. llama.cpp loads + # one model per process and the already-running Qwen3.8-27B chat model + # isn't embedding-trained, so this is a second, separate llama-server + # instance rather than adding --embeddings to the chat one — see + # docs/research/litellm-knowledgebase.md #3. bge-small-en-v1.5 outputs + # 384-dim vectors — use that as the Qdrant collection size. + # ponytail: CPU image, no GPU devices — a 33M-param embedding model is + # fast enough on CPU and this avoids VRAM contention with llama-server's + # 27B chat model on the same GPU. + embedding-server: + image: ghcr.io/ggml-org/llama.cpp:server + container_name: embedding-server + volumes: + - models:/models + command: > + -m /models/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf} + --host 0.0.0.0 + --port 8081 + --embeddings + --pooling mean + restart: unless-stopped + networks: [ai-stack] # Vector store for OmniRoute's memory feature — wired up in the OmniRoute # dashboard as a memory service, not via static config here. Not published