From 9e9cac254b6e9825900a973eb3c086fd623a8b86 Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Thu, 3 Sep 2026 22:19:21 +0200 Subject: [PATCH] feat(stack): drop qdrant and embedding-server, use OmniRoute's built-in memory OmniRoute's memory feature is self-contained: its bundled sqlite-vec vector store plus a local ONNX embedding model (Transformers.js, ~400MB, fetched into the omniroute-data volume on first use) replace the external qdrant + bge-small-en-v1.5 embedding-server pair, which was never wired up in the dashboard. Two fewer containers, no second GGUF download, no EMBEDDING_MODEL_FILE var. Memory stays opt-in via the dashboard (Settings -> Memory, transformers source); nothing here changes the gateway's static config. --- .env.example | 6 ------ README.md | 2 +- docker-compose.yml | 48 +--------------------------------------------- 3 files changed, 2 insertions(+), 54 deletions(-) diff --git a/.env.example b/.env.example index 7f45eb0..95d1335 100644 --- a/.env.example +++ b/.env.example @@ -28,12 +28,6 @@ LLAMA_GPU_LAYERS=999 # of dropping context: --cache-type-k/v q8_0 roughly halves it. LLAMA_CTX_SIZE=131072 -# --- embedding-server (OmniRoute memory) --- -# bge-small-en-v1.5, CPU-only, 384-dim output — see docker-compose.yml's -# embedding-server service comment for why this is a second model/process -# rather than reusing llama-server. -EMBEDDING_MODEL_FILE=bge-small-en-v1.5-q8_0.gguf - # --- Lazytainer --- # Seconds of inactivity before llama-server is stopped. 900 = 15 min. LAZYTAINER_INACTIVE_TIMEOUT=900 diff --git a/README.md b/README.md index 3153876..9942f3c 100644 --- a/README.md +++ b/README.md @@ -37,4 +37,4 @@ The gateway also fronts SearXNG-backed web search — see `docs/research/litellm ## What's not here - **Open WebUI** — this stack has no chat UI; every client is a coding CLI. Removed rather than kept idle. -- **Gateway-level knowledgebase/memory** (`litellm-pgvector`, `pgvector-db`, a dedicated embedding model) — removed as unwanted, unrelated to OmniRoute's own lack of parity with it (see issue #31's #34). Superseded by OmniRoute's own memory feature, backed by `qdrant` and `embedding-server` (bge-small-en-v1.5, CPU-only) — both bare services, wired up as a memory provider by hand in the dashboard, no static routing config here. +- **Gateway-level knowledgebase/memory** (`litellm-pgvector`, `pgvector-db`, a dedicated embedding model) — removed as unwanted, unrelated to OmniRoute's own lack of parity with it (see issue #31's #34). Superseded by OmniRoute's own built-in memory feature (opt-in via the dashboard, Settings → Memory): vector store is its bundled `sqlite-vec`, embeddings are a local ONNX model (Transformers.js, ~400MB, downloaded into the `omniroute-data` volume on first use) — no external services, no static config here. diff --git a/docker-compose.yml b/docker-compose.yml index 4199b09..d353aab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,50 +54,7 @@ services: test -f /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} && echo "already downloaded, skipping" || curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} - https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}; - test -f /models/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf} && - echo "already downloaded, skipping" || - curl -L --fail --create-dirs -o /models/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf} - https://huggingface.co/ggml-org/bge-small-en-v1.5-Q8_0-GGUF/resolve/main/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf} - - # Dedicated embedding model for OmniRoute's memory feature. llama.cpp loads - # one model per process and the already-running Qwen3.8-27B chat model - # isn't embedding-trained, so this is a second, separate llama-server - # instance rather than adding --embeddings to the chat one — see - # docs/research/litellm-knowledgebase.md #3. bge-small-en-v1.5 outputs - # 384-dim vectors — use that as the Qdrant collection size. - # ponytail: CPU image, no GPU devices — a 33M-param embedding model is - # fast enough on CPU and this avoids VRAM contention with llama-server's - # 27B chat model on the same GPU. - embedding-server: - image: ghcr.io/ggml-org/llama.cpp:server - container_name: embedding-server - volumes: - - models:/models - command: > - -m /models/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf} - --host 0.0.0.0 - --port 8081 - --embeddings - --pooling mean - restart: unless-stopped - networks: [ai-stack] - - # Vector store for OmniRoute's memory feature — wired up in the OmniRoute - # dashboard as a memory service, not via static config here. Not published - # to the host: only OmniRoute (same ai-stack network) talks to it. - qdrant: - image: qdrant/qdrant:latest - container_name: qdrant - volumes: - - qdrant-data:/qdrant/storage - restart: unless-stopped - networks: [ai-stack] - healthcheck: - test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/localhost/6333'"] - interval: 10s - timeout: 5s - retries: 5 + https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} # Replaces litellm — see issue #31 (wayfinder map) for the full migration # rationale/findings. No static config.yaml equivalent: provider routing @@ -110,8 +67,6 @@ services: depends_on: llama-server: condition: service_started - qdrant: - condition: service_healthy volumes: - omniroute-data:/app/data env_file: .env @@ -191,4 +146,3 @@ networks: volumes: models: omniroute-data: - qdrant-data: