docs(coding-cli-setup): split per-CLI docs into their own files

Move docs/coding-cli-setup.md to docs/coding-cli-setup/ with one file per CLI (claude-code, kimi-cli, opencode, qwen-code) plus a shared index.md for the gateway intro, tool-calling risk note, and summary table. Also fixes the qwen-code doc: context sizes are per-slot (LLAMA_CTX_SIZE / LLAMA_PARALLEL), not raw LLAMA_CTX_SIZE (same fix applied to OpenCode's limit.context); documents the fastModel classifier provider and its own context math; adds the omniroute-search MCP server (SearXNG web search) and Auto Mode permissions tuning that were missing from the original qwen-code section. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
fix(update.sh): stop config sync loop from dying silently on a missing key
2026-09-08 12:30:08 +02:00 · 2026-09-07 20:09:31 +02:00 · 2026-09-07 20:04:56 +02:00 · 2026-09-07 20:01:20 +02:00 · 2026-09-07 19:54:38 +02:00 · 2026-09-07 19:40:51 +02:00
39 changed files with 2577 additions and 1966 deletions
@@ -6,56 +6,143 @@

 # --- llama.cpp / model ---
 LLAMA_MODEL_FILE=Qwen3.8-27B-UD-Q4_K_XL.gguf
+# 999 = every layer on GPU (this model is dense, not MoE, and already fits
+# fully in 32GB VRAM — see docs/research/qwen3.8-27b-quant.md). Lower this
+# to leave that many fewer layers on GPU and push the rest to CPU/system RAM
+# if something else is contending for VRAM — llama.cpp has no separate
+# "RAM offload" flag, --n-gpu-layers *is* the RAM-offload knob for a dense
+# model. Don't reach for --n-cpu-moe/--cpu-moe/--override-tensor "exps" —
+# those target Mixture-of-Experts models (e.g. Qwen3.8-2.4T-A95B), not this
+# one, and are no-ops here.
+# There's no separate "then SSD" tier to enable either: llama.cpp mmaps the
+# model file by default (no --no-mmap here), so if GPU+RAM ever can't hold
+# the working set, the OS pages the rest in from disk automatically — an
+# implicit, slow last resort, not a config knob. An explicit tiered SSD
+# offload has been an open llama.cpp feature request since 2025 (still
+# unimplemented): https://github.com/ggml-org/llama.cpp/discussions/12507
 LLAMA_GPU_LAYERS=999
-# 65536 (64K) fits comfortably in 32GB VRAM alongside the model weights.
-# Raise toward 131072 if you need more context; see docs/research/qwen3.8-27b-quant.md
-# for the VRAM math at larger context sizes.
-LLAMA_CTX_SIZE=65536
-
-# --- Open WebUI ---
-WEBUI_PORT=8008
-# Minted automatically by ./scripts/update.sh — leave blank. Manual fallback:
-# docs/proxy-key-onboarding.md.
-OPENWEBUI_LITELLM_KEY=
+# 262144 = this model's true max (max_position_embeddings in Qwen/Qwen3.8-27B's
+# config.json) — the largest --ctx-size llama.cpp will even accept for it.
+# fp16 KV cache at full context would be ~16GB, on top of 17.6GB weights =
+# ~33.6GB, which does NOT fit the 32GB R9700 on its own. docker-compose.yml
+# now runs --cache-type-k/v q8_0, which roughly halves KV memory (~8GB at
+# this size) — total ~25.6GB, ~6GB headroom, the same footprint the old
+# 131072 fp16 setting used. See docs/research/qwen3.8-27b-quant.md.
+LLAMA_CTX_SIZE=262144
+# Concurrent request slots. Was implicitly 4 (llama.cpp's compiled-in
+# default) with no flag set — under concurrent subagent fan-out, 4 requests
+# split the same GPU compute, so a large-context prefill can queue behind
+# others long enough to blow past OmniRoute's stream-idle timeout, which then
+# cancels the request (see issue-tracker notes on the timeout/cancel loop).
+# Dropped to 2 so each slot gets more compute and finishes prefill sooner;
+# raise back toward 4 if throughput (not latency) becomes the bottleneck
+# instead. Each slot gets LLAMA_CTX_SIZE / LLAMA_PARALLEL tokens of context —
+# real sessions have hit ~66K tokens, so don't drop LLAMA_CTX_SIZE without
+# checking that per-slot number stays comfortably above observed usage.
+LLAMA_PARALLEL=2

 # --- Lazytainer ---
 # Seconds of inactivity before llama-server is stopped. 900 = 15 min.
 LAZYTAINER_INACTIVE_TIMEOUT=900

-# --- Embedding model (knowledgebase, see docs/memory-knowledgebase.md) ---
-EMBEDDING_MODEL_FILE=nomic-embed-text-v1.5.Q8_0.gguf
-
 # --- SearXNG web search (see docs/research/litellm-searxng-search.md) ---
 # Resolved automatically by ./scripts/update.sh from search.home on this
 # host — leave blank. Only set by hand if that resolution fails (e.g.
 # search.home isn't a static DHCP reservation and its IP drifted).
 SEARXNG_LAN_IP=

-# --- LiteLLM proxy (see docs/proxy-key-onboarding.md, docs/network-access.md) ---
-LITELLM_PORT=4000
+# --- OmniRoute gateway (see docs/proxy-key-onboarding.md, docs/network-access.md) ---
+# OMNIROUTE_PORT is the host-published port (reverse-proxied by NPM) — kept
+# at 4000, same as the old LiteLLM setup, so existing NPM/firewall config
+# doesn't need to change. It's mapped via plain Docker port publishing onto
+# API_PORT, omniroute's own container-internal port (left at its default,
+# not reconfigured to match). The dashboard (DASHBOARD_PORT) is never
+# published at all — see docker-compose.yml's omniroute service comment.
+OMNIROUTE_API_PORT=20129
+OMNIROUTE_DASHBOARD_PORT=20128
+# SSE inactivity timeout before OmniRoute gives up on a streaming request and
+# cancels it (which cancels the matching llama-server task too). 180s gives
+# contended prefill (see LLAMA_PARALLEL above) room to produce a first token.
+OMNIROUTE_STREAM_IDLE_TIMEOUT_MS=180000
 # Random values, filled in automatically by ./scripts/update.sh — leave
-# blank. LITELLM_SALT_KEY encrypts stored data; do not change it after the
-# first run (existing encrypted data becomes unreadable if you do).
-LITELLM_MASTER_KEY=
-LITELLM_SALT_KEY=
-LITELLM_DB_PASSWORD=
-# Backs litellm's router state/rate-limits/budgets/cache invalidation
-# (the redis service). Random value, filled in automatically — leave blank.
-REDIS_PASSWORD=
-# Admin UI login (https://<proxy>/ui). Without these, LiteLLM falls back to
-# username "admin" / password = LITELLM_MASTER_KEY — set these instead so the
-# master key never has to be typed into the browser. UI_PASSWORD is filled
-# in automatically by ./scripts/update.sh if blank.
-UI_USERNAME=admin
-UI_PASSWORD=
+# blank. Bootstrap dashboard admin password (log in at the dashboard port,
+# change it there afterwards — this is only the first-boot value):
+OMNIROUTE_INITIAL_PASSWORD=
+# Signs dashboard session cookies:
+OMNIROUTE_JWT_SECRET=
+# Encrypts API key values at rest in omniroute's SQLite DB:
+OMNIROUTE_API_KEY_SECRET=
+# Encrypts the whole SQLite DB at rest. Do not change after first run —
+# existing encrypted data becomes unreadable if you do (same caveat as
+# LiteLLM's old LITELLM_SALT_KEY):
+OMNIROUTE_STORAGE_ENCRYPTION_KEY=
+# Per-deployment salts — random is fine, just needs to be stable:
+OMNIROUTE_MACHINE_ID_SALT=
+OMNIROUTE_CLI_SALT=
+# Required (production) — shared secret for the internal Codex Responses
+# WebSocket bridge. Random value, filled in automatically:
+OMNIROUTE_WS_BRIDGE_SECRET=
+# Per-workload virtual keys (one per client that calls the gateway) have no
+# scripted /key/generate equivalent yet — omniroute's key-creation endpoint
+# needs a dashboard login session, not a static bearer key (see issue #37).
+# Mint them by hand in the dashboard, add a KEY=value line here per workload
+# as you onboard one. See docs/proxy-key-onboarding.md.

-# --- Knowledgebase (pgvector + litellm-pgvector, see docs/memory-knowledgebase.md) ---
-# Random value, filled in automatically by ./scripts/update.sh — leave blank.
-PGVECTOR_DB_PASSWORD=
-# Auth key litellm-pgvector requires on its own API (its SERVER_API_KEY).
-# Random value, filled in automatically by ./scripts/update.sh — leave blank.
-LITELLM_PGVECTOR_API_KEY=
-# A virtual key litellm-pgvector uses to call back into litellm for
-# embeddings. Minted automatically by ./scripts/update.sh — leave blank.
-# Manual fallback: docs/proxy-key-onboarding.md.
-LITELLM_PGVECTOR_EMBEDDING_KEY=
+# --- ComfyUI (local image generation, see issue #38 wayfinder map) ---
+# yurisasc/comfyui-rocm7.1 manages GPU-group access via these GID/UID env
+# vars rather than relying solely on docker-compose.yml's group_add.
+# Resolved automatically from the host by ./scripts/update.sh — leave blank.
+COMFYUI_PUID=
+COMFYUI_PGID=
+
+# Shared by every GPU-touching service (llama-server, llama-server-fast,
+# comfyui) for group_add: — resolved to real host GIDs by ./scripts/update.sh
+# rather than left as plain group names in docker-compose.yml, because Docker
+# resolves a *named* group_add entry against the container's own /etc/group,
+# not the host's, and fails unpredictably when the image doesn't define one
+# (worse with multiple GPU services racing on the same lookup at once — see
+# docs/research/rocm-gpu-pin-and-render-group.md and issue #5). Leave blank.
+HOST_VIDEO_GID=
+HOST_RENDER_GID=
+
+# --- llama.cpp / fast model (second, always-resident instance — see
+# docs/research/fast-model-choice.md and issue #44) ---
+# Qwen3-4B-Instruct-2507: architecturally non-thinking (never emits
+# <think> blocks, unlike Qwen3-1.7B/0.6B which need a per-call toggle) —
+# picked specifically so it stays fast enough for qwen-code's Auto Mode
+# classifier (Stage 1 wants ~300ms). Same publisher (unsloth) as the main
+# model for consistency.
+LLAMA_FAST_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf
+# Same reasoning as LLAMA_GPU_LAYERS above — full GPU offload, this model
+# is dense too.
+LLAMA_FAST_GPU_LAYERS=999
+# --ctx-size is the TOTAL across every LLAMA_FAST_PARALLEL slot, not per
+# request — same halving already called out for the main model above.
+# Was PARALLEL=2, silently halving this to 4096/slot — too small: a real
+# classifier call (hints + environment + recent tool-call history) hit
+# "exceeds the available context size (4096 tokens)" in practice, which
+# qwen-code surfaces as "Auto Mode couldn't classify this action
+# (Classifier stage 1 unavailable)" — see issue #5. Fixed by dropping to
+# a single slot instead of raising ctx-size (no extra VRAM, and this
+# service doesn't need concurrent classifier calls the way the main
+# model needs concurrent chat sessions) — the full 8192 now goes to the
+# one slot. If hints.allow/softDeny/hardDeny ever approach their
+# 50-entries-each ceiling, raise LLAMA_FAST_CTX_SIZE instead — qwen-code
+# caps those at 200 chars x 150 entries plus 40,000 chars of
+# historical-action context, which can exceed 8192 tokens worst-case.
+LLAMA_FAST_CTX_SIZE=8192
+LLAMA_FAST_PARALLEL=1
+
+# --- ComfyUI diffusion model (Qwen-Image, FP8 — see docs/research/
+# image-generation-model-choice.md and issue #42) ---
+# Three files: diffusion weights, text encoder, VAE — all from the official
+# Comfy-Org FP8 split, chosen specifically because it's the only candidate
+# with a ComfyUI workflow pre-validated on this exact GPU (gfx1201/R9700).
+COMFYUI_DIFFUSION_MODEL_FILE=qwen_image_fp8_e4m3fn.safetensors
+COMFYUI_TEXT_ENCODER_FILE=qwen_2.5_vl_7b_fp8_scaled.safetensors
+COMFYUI_VAE_FILE=qwen_image_vae.safetensors
+
+# --- RAG databases (qdrant + neo4j, see wayfinder notes) ---
+# No auth on qdrant (its default) — same trust boundary as llama-server:
+# ai-stack is not exposed off-box. Random, filled in automatically:
+NEO4J_PASSWORD=
@@ -4,3 +4,5 @@
 # to be committed to this repo.
 data/
 .leankg/
+.cache/
+.qwen/temp
@@ -10,4 +10,4 @@ Single-context: `CONTEXT.md` + `docs/adr/` at the repo root. See `docs/agents/do

 ### Deploying changes

-The running stack lives on a separate box (the R9700 server), not wherever this repo is being edited. After **any** change to `docker-compose.yml`, `litellm-config.yaml`, `.env.example`, or a script under `scripts/`, commit/push it, then run `./scripts/update.sh` on the server to apply it — don't just describe the change as done. If this session doesn't have shell access to the server, say so explicitly and tell the user to run it themselves rather than leaving it unsaid.
+The running stack lives on a separate box (the R9700 server), not wherever this repo is being edited. After **any** change to `docker-compose.yml`, `.env.example`, or a script under `scripts/`, commit/push it, then run `./scripts/update.sh` on the server to apply it — don't just describe the change as done. If this session doesn't have shell access to the server, say so explicitly and tell the user to run it themselves rather than leaving it unsaid.
@@ -1,6 +1,6 @@
 # LLM-Server

-Local AI inference stack: llama.cpp (ROCm) serving Qwen3.8-27B on an AMD Radeon AI PRO R9700, fronted by Open WebUI (RAG + Memory via Qdrant), with Lazytainer auto-suspending the inference container when idle.
+Local AI inference stack: llama.cpp (ROCm) serving Qwen3.8-27B on an AMD Radeon AI PRO R9700, fronted by the OmniRoute AI gateway, with Lazytainer auto-suspending the inference container when idle.

 See the wayfinder map ([issue #1](https://git.arthurerlich.de/haylan/LLM-Server/issues/1)) for the full architecture rationale and open questions.

@@ -10,26 +10,26 @@ See the wayfinder map ([issue #1](https://git.arthurerlich.de/haylan/LLM-Server/
 ./scripts/update.sh
 ```

-`update.sh` creates `.env` from `.env.example` if missing, fills in every secret and per-workload virtual key it can generate itself (random secrets via `openssl`, `OPENWEBUI_LITELLM_KEY`/`LITELLM_PGVECTOR_EMBEDDING_KEY` minted through LiteLLM's own `/key/generate` API, `SEARXNG_LAN_IP` resolved from `search.home` on this host), downloads both model GGUFs into the `models` volume if they're not there yet, then pulls/builds/brings up the whole stack. Safe to re-run any time — it only fills in what's still blank, skips models already downloaded, and only recreates what changed. See [`docs/proxy-key-onboarding.md`](docs/proxy-key-onboarding.md) if a key mint fails and needs doing by hand.
+`update.sh` creates `.env` from `.env.example` if missing, fills in every random secret it can generate itself (via `openssl`, `SEARXNG_LAN_IP` resolved from `search.home` on this host), downloads the model GGUF into the `models` volume if it's not there yet, then pulls/builds/brings up the whole stack. Safe to re-run any time — it only fills in what's still blank, skips the model if already downloaded, and only recreates what changed.

- Open WebUI: `http://<this-machine>:3000` locally, or `ai.home` / `ai.haylan.ch` once routed through Nginx Proxy Manager — see [`docs/network-access.md`](docs/network-access.md). First signup becomes the admin account (`WEBUI_AUTH` is on).
- llama.cpp's own API is internal-only now — everything routes through the AI proxy below.
+llama.cpp's own API is internal-only — everything routes through the AI gateway below.

-Pointing Claude Code CLI, Kimi CLI, or OpenCode CLI at the local endpoint: see [`docs/coding-cli-setup.md`](docs/coding-cli-setup.md).
+Pointing Claude Code CLI, Kimi CLI, OpenCode CLI, or Qwen Code CLI at the local endpoint: see [`docs/coding-cli-setup/`](docs/coding-cli-setup/index.md).

 **Known risk**: Qwen3.8-27B's tool-calling reliability against llama.cpp's Anthropic shim is not yet verified (open upstream parser bugs against its model lineage) — see `docs/research/qwen3.8-27b-tool-calling.md`.

-## AI proxy (LiteLLM)
+## AI gateway (OmniRoute)

-An [AI gateway/proxy](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) fronts llama.cpp: per-workload virtual keys, usage tracking, and a shadow cost estimate ("what this would have cost on Claude Sonnet 5"). `./scripts/update.sh` handles `LITELLM_MASTER_KEY`/`LITELLM_SALT_KEY` and every other secret (see `.env.example`).
+An [AI gateway/proxy](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) fronts llama.cpp: per-workload API keys and usage tracking. As of [issue #31](https://git.arthurerlich.de/haylan/LLM-Server/issues/31) this is [OmniRoute](https://github.com/diegosouzapw/OmniRoute), replacing the original LiteLLM setup. `./scripts/update.sh` handles most of OmniRoute's secrets (see `.env.example`); per-workload API keys still need minting by hand in the dashboard — see [`docs/proxy-key-onboarding.md`](docs/proxy-key-onboarding.md).

- Proxy API: `http://<this-machine>:4000/v1` locally, or `proxy.ai.home` / `proxy.ai.haylan.ch` once routed through NPM — see [`docs/network-access.md`](docs/network-access.md).
- Admin UI (`/ui`, key/budget management): LAN-only — see `docs/network-access.md`.
+- Gateway API: `http://<this-machine>:${OMNIROUTE_PORT:-4000}/v1` locally, or `proxy-ai.home` / `proxy-ai.haylan.ch` once routed through NPM — see [`docs/network-access.md`](docs/network-access.md).
+- Dashboard (key/provider management): LAN/host-only, never published to the internet — see `docs/network-access.md`.
 - Issuing a key for a new workload: [`docs/proxy-key-onboarding.md`](docs/proxy-key-onboarding.md).
- Request priority across workloads: [`docs/proxy-request-priority.md`](docs/proxy-request-priority.md).

-Open WebUI and the coding CLIs (see [`docs/coding-cli-setup.md`](docs/coding-cli-setup.md)) route through the proxy now — llama-server has no published host port anymore. **Not yet verified**: none of this has been smoke-tested on real hardware (LiteLLM's priority scheduler in particular is beta — see `docs/proxy-request-priority.md`) — see [issue #17](https://git.arthurerlich.de/haylan/LLM-Server/issues/17).
+Coding CLIs (see [`docs/coding-cli-setup/`](docs/coding-cli-setup/index.md)) route through the gateway — llama-server has no published host port. **Not yet verified**: none of this has been smoke-tested on real hardware yet — see [issue #31](https://git.arthurerlich.de/haylan/LLM-Server/issues/31)'s tickets for the open items (provider registration, per-workload key minting).

-### Web search, knowledgebase, and memory
+**Note on this choice**: OmniRoute's own docs (`docs/security/STEALTH_GUIDE.md`, `MITM-TPROXY-DECRYPT.md`, `PUBLIC_CREDS.md` in its repo) describe shipped features for evading AI-provider client detection, system-wide HTTPS interception via a locally-installed root CA, and hiding credentials from secret scanners. None of that is used by this stack's configuration, but it's a real characteristic of the upstream project — see issue #31's Notes for the full research trail before extending this integration further.

-The gateway also fronts SearXNG-backed web search and a pgvector-backed knowledgebase (loaded with `data/memory.md` / `data/claude-legacy-memory.md`), wired at the LiteLLM layer so every client gets them, not just Open WebUI — see [`docs/memory-knowledgebase.md`](docs/memory-knowledgebase.md). **Not yet verified on real hardware** — see [issue #24](https://git.arthurerlich.de/haylan/LLM-Server/issues/24).
+### Web search
+
+The gateway also fronts SearXNG-backed web search — see `docs/research/litellm-searxng-search.md` for the original research (still applicable — same standalone-endpoint pattern, see issue #31's #35).
@@ -5,12 +5,20 @@ services:
    devices:
      - /dev/kfd
      - /dev/dri
+    # Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in
+    # .env.example and docs/research/rocm-gpu-pin-and-render-group.md.
    group_add:
-      - video
-      - render
+      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
+      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
+    # Caps this process's HIP hardware-queue allocation — works around
+    # ROCm/ROCm#5706 (GPU pinned at 100%/boost-clock whenever two
+    # concurrent HIP contexts touch this card, confirmed on real hardware
+    # against llama-server-fast below). See the research doc above.
+    environment:
+      - GPU_MAX_HW_QUEUES=1
    volumes:
      - models:/models
    command: >
@@ -18,10 +26,14 @@ services:
      --host 0.0.0.0
      --port 8080
      --n-gpu-layers ${LLAMA_GPU_LAYERS:-999}
-      --ctx-size ${LLAMA_CTX_SIZE:-65536}
+      --ctx-size ${LLAMA_CTX_SIZE:-262144}
+      --parallel ${LLAMA_PARALLEL:-2}
+      --flash-attn on
+      --cache-type-k q8_0
+      --cache-type-v q8_0
      --jinja
-    # No published host port: llama-server is reached only via the litellm
-    # proxy on the ai-stack docker network now — see issue #15. Its
+    # No published host port: llama-server is reached only via the omniroute
+    # gateway on the ai-stack docker network now — see issue #15. Its
    # unauthenticated API no longer needs to be LAN-reachable directly.
    expose:
      - "8080"
@@ -35,41 +47,54 @@ services:
      - "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
      - "lazytainer.group.llamaserver.minPacketThreshold=2"

-  embedding-server:
+  llama-server-fast:
    image: ghcr.io/ggml-org/llama.cpp:server-rocm
-    container_name: embedding-server
+    container_name: llama-server-fast
    devices:
      - /dev/kfd
      - /dev/dri
    group_add:
-      - video
-      - render
+      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
+      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
+    # See llama-server's identical setting above — same fix, same bug.
+    environment:
+      - GPU_MAX_HW_QUEUES=1
    volumes:
      - models:/models
    command: >
-      -m /models/${EMBEDDING_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf}
+      -m /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
      --host 0.0.0.0
      --port 8080
-      --embeddings
-      --pooling mean
-      --n-gpu-layers 999
-      --ctx-size 8192
-    # A dedicated embedding model — the chat model isn't embedding-trained
-    # and llama.cpp serves one model per process, so this is a second small
-    # instance, not a mode switch on llama-server. See
-    # docs/research/litellm-knowledgebase.md. Small enough (~150MB Q8) to
-    # run alongside the chat model's ~19.6GB in the R9700's 32GB VRAM.
+      --n-gpu-layers ${LLAMA_FAST_GPU_LAYERS:-999}
+      --ctx-size ${LLAMA_FAST_CTX_SIZE:-8192}
+      --parallel ${LLAMA_FAST_PARALLEL:-1}
+      --flash-attn on
+      --cache-type-k q8_0
+      --cache-type-v q8_0
+      --reasoning off
+      --jinja
+    # Second, always-resident llama.cpp instance — small non-thinking model
+    # used as qwen-code's Auto Mode classifier fastModel, alongside the main
+    # 27B instance above. See docs/research/fast-model-choice.md and #44.
+    # Same ai-stack-only pattern as llama-server: no published host port.
+    expose:
+      - "8080"
    restart: unless-stopped
    networks: [ai-stack]
+    labels:
+      - "lazytainer.group.llamaserverfast.sleepMethod=stop"
+      - "lazytainer.group.llamaserverfast.ports=8080"
+      - "lazytainer.group.llamaserverfast.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
+      - "lazytainer.group.llamaserverfast.minPacketThreshold=2"

  # ponytail: one-off downloader, not a standing service — run via
  # `docker compose --profile tools run --rm downloader`. Folded into
  # scripts/update.sh, which runs this every time; the `test -f` guard is
  # what makes that safe to re-run without re-downloading. Keeps the model
-  # file inside the named `models` volume instead of a host bind-mount.
+  # files inside the named `models` volume instead of a host bind-mount.
  downloader:
    image: curlimages/curl:latest
    profiles: ["tools"]
@@ -86,10 +111,9 @@ services:
        curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
        https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}

-  # ponytail: same one-off pattern as `downloader`, for the embedding model —
-  # run via `docker compose --profile tools run --rm downloader-embedding`,
-  # also folded into scripts/update.sh.
-  downloader-embedding:
+  # Same test -f guard pattern as downloader above — fetches the second,
+  # smaller model for llama-server-fast. See issue #44.
+  downloader-fast:
    image: curlimages/curl:latest
    profiles: ["tools"]
    user: root
@@ -98,192 +122,167 @@ services:
    entrypoint: ["sh", "-c"]
    command:
      - >
-        test -f /models/${EMBEDDING_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf} &&
+        test -f /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} &&
        echo "already downloaded, skipping" ||
-        curl -L --fail --create-dirs -o /models/${EMBEDDING_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf}
-        https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/${EMBEDDING_MODEL_FILE:-nomic-embed-text-v1.5.Q8_0.gguf}
+        curl -L --fail --create-dirs -o /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
+        https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}

-  qdrant:
-    image: qdrant/qdrant:latest
-    container_name: qdrant
+  # Fetches the three Qwen-Image FP8 files ComfyUI needs (diffusion model,
+  # text encoder, VAE) — same test -f guard pattern as downloader/
+  # downloader-fast above. See docs/research/image-generation-model-choice.md
+  # and issue #42.
+  #
+  # ponytail: target paths assume ComfyUI's standard models/ layout under
+  # BASE_STORAGE_PATH (/storage) — same "not independently confirmed against
+  # the image's Dockerfile" caveat already flagged on the comfyui service
+  # below. If ComfyUI doesn't pick these up, check its actual models root
+  # first.
+  downloader-comfyui:
+    image: curlimages/curl:latest
+    profiles: ["tools"]
+    user: root
    volumes:
-      - qdrant-data:/qdrant/storage
-    restart: unless-stopped
-    networks: [ai-stack]
-    healthcheck:
-      test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/localhost/6333'"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
+      - comfyui-data:/storage
+    entrypoint: ["sh", "-c"]
+    command:
+      - >
+        mkdir -p /storage/models/diffusion_models /storage/models/text_encoders /storage/models/vae &&
+        (test -f /storage/models/diffusion_models/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen_image_fp8_e4m3fn.safetensors} &&
+        echo "diffusion model already downloaded, skipping" ||
+        curl -L --fail --create-dirs -o /storage/models/diffusion_models/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen_image_fp8_e4m3fn.safetensors}
+        https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/diffusion_models/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen_image_fp8_e4m3fn.safetensors}) &&
+        (test -f /storage/models/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen_2.5_vl_7b_fp8_scaled.safetensors} &&
+        echo "text encoder already downloaded, skipping" ||
+        curl -L --fail --create-dirs -o /storage/models/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen_2.5_vl_7b_fp8_scaled.safetensors}
+        https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen_2.5_vl_7b_fp8_scaled.safetensors}) &&
+        (test -f /storage/models/vae/${COMFYUI_VAE_FILE:-qwen_image_vae.safetensors} &&
+        echo "vae already downloaded, skipping" ||
+        curl -L --fail --create-dirs -o /storage/models/vae/${COMFYUI_VAE_FILE:-qwen_image_vae.safetensors}
+        https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/vae/${COMFYUI_VAE_FILE:-qwen_image_vae.safetensors})

-  open-webui:
-    image: ghcr.io/open-webui/open-webui:main
-    container_name: open-webui
-    depends_on:
-      qdrant:
-        condition: service_healthy
-      litellm:
-        condition: service_healthy
-    volumes:
-      - openwebui-data:/app/backend/data
-    env_file: .env
+  # Local image generation — see issue #38 (wayfinder map). yurisasc's image
+  # is gfx1201-tuned specifically (R9700's arch), unlike the official/AMD
+  # ComfyUI image which doesn't pin RDNA4 support — see
+  # docs/research/image-generation-options.md.
+  comfyui:
+    image: yurisasc/comfyui-rocm7.1:latest
+    container_name: comfyui
+    devices:
+      - /dev/kfd
+      - /dev/dri
+    # Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in
+    # .env.example and docs/research/rocm-gpu-pin-and-render-group.md.
+    group_add:
+      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
+      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
+    security_opt:
+      - seccomp=unconfined
+    ipc: host
    environment:
-      - WEBUI_AUTH=True
-      # Routed through the litellm proxy, not llama-server directly — see issue #15.
-      # OPENAI_API_KEY must be a virtual key created for Open WebUI per
-      # docs/proxy-key-onboarding.md (name it "openwebui"), set as
-      # OPENWEBUI_LITELLM_KEY in .env.
-      - OPENAI_API_BASE_URL=http://litellm:4000/v1
-      - OPENAI_API_KEY=${OPENWEBUI_LITELLM_KEY}
-      - VECTOR_DB=qdrant
-      - QDRANT_URI=http://qdrant:6333
+      - HSA_OVERRIDE_GFX_VERSION=12.0.1
+      - PYTORCH_ROCM_ARCH=gfx1201
+      # This image also wants GID env vars directly (its own README asks
+      # for both these and group_add above) — same HOST_VIDEO_GID/
+      # HOST_RENDER_GID resolved by scripts/update.sh, shared with
+      # llama-server/llama-server-fast now instead of comfyui-only vars.
+      - PUID=${COMFYUI_PUID}
+      - PGID=${COMFYUI_PGID}
+      - VIDEO_GID=${HOST_VIDEO_GID}
+      - RENDER_GID=${HOST_RENDER_GID}
+      - BASE_STORAGE_PATH=/storage
+    volumes:
+      - comfyui-data:/storage
+    # ponytail: exact internal storage path taken from the image's own
+    # BASE_STORAGE_PATH env var, not independently confirmed against its
+    # Dockerfile — if models/workflows don't persist across a recreate,
+    # check this against the image's actual entrypoint first.
+    #
+    # Published host port (unlike llama-server's ai-stack-only pattern):
+    # ComfyUI's own UI is meant to be reachable directly too, for a planned
+    # external nginx reverse-proxy route to comfy.home — not just through
+    # OmniRoute. Still also reachable at http://comfyui:8188 internally on
+    # ai-stack, which is the URL to register as OmniRoute's comfyui
+    # provider (dashboard or POST /api/providers, per docs/proxy-key-onboarding.md
+    # — same undocumented-in-repo manual flow already used for llama-server).
    ports:
-      - "${WEBUI_PORT:-8008}:8080"
+      - "8138:8188"
    restart: unless-stopped
    networks: [ai-stack]

-  litellm:
-    image: ghcr.io/berriai/litellm:main-stable
-    container_name: litellm
+  # Replaces litellm — see issue #31 (wayfinder map) for the full migration
+  # rationale/findings. No static config.yaml equivalent: provider routing
+  # (llama-server, searxng-search) is registered once through the dashboard
+  # or POST /api/providers after first boot, not checked into this repo —
+  # see docs/proxy-key-onboarding.md.
+  omniroute:
+    image: diegosouzapw/omniroute:latest
+    container_name: omniroute
    depends_on:
-      litellm-db:
-        condition: service_healthy
      llama-server:
        condition: service_started
-      redis:
-        condition: service_healthy
+      llama-server-fast:
+        condition: service_started
    volumes:
-      - ./litellm-config.yaml:/app/config.yaml:ro
-    # LITELLM_MASTER_KEY / LITELLM_SALT_KEY come straight from .env via env_file
-    # (names match what litellm reads). LITELLM_SALT_KEY must not change after
-    # first run — see .env.example.
+      - omniroute-data:/app/data
    env_file: .env
    environment:
-      - DATABASE_URL=postgresql://litellm:${LITELLM_DB_PASSWORD}@litellm-db:5432/litellm
-      # Setting these is all LiteLLM needs to use Redis for router state,
-      # rate limits/budgets, and cache invalidation — no extra config.yaml
-      # block required. See https://docs.litellm.ai/docs/proxy/caching.
-      - REDIS_HOST=redis
-      - REDIS_PORT=6379
-      - REDIS_PASSWORD=${REDIS_PASSWORD}
-    # The litellm container only joins the ai-stack bridge network, which has
-    # no visibility into the LAN's mDNS/local-DNS names — search.home won't
-    # resolve without this. Set SEARXNG_LAN_IP in .env to its stable LAN IP
-    # (static DHCP reservation recommended). See docs/research/litellm-searxng-search.md.
+      # Split-port mode: dashboard and API are fully separate ports (unlike
+      # LiteLLM's single :4000 for both /v1 and /ui) — both published
+      # directly below, unlike the old :4000-only host mapping.
+      - API_HOST=0.0.0.0
+      - API_PORT=${OMNIROUTE_API_PORT:-20129}
+      - DASHBOARD_PORT=${OMNIROUTE_DASHBOARD_PORT:-20128}
+      # Required to register llama-server/searxng-search as providers —
+      # their base URLs are LAN/container-internal addresses, blocked by
+      # default (SSRF guard against public-provider spoofing).
+      - OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS=true
+      - OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS=true
+      # Required (production) per docs/reference/ENVIRONMENT.md — shared
+      # secret for the internal Codex Responses WebSocket bridge. Missed on
+      # first pass; docker-compose config validated fine without it, but
+      # the docs are explicit this one's required, not optional.
+      - OMNIROUTE_WS_BRIDGE_SECRET=${OMNIROUTE_WS_BRIDGE_SECRET}
+      # Default heap (1024MB) is dashboard-only sized per OmniRoute's own
+      # Docker guide — every client here is a coding CLI, which needs the
+      # larger figure the guide recommends. Paired with mem_limit below.
+      - OMNIROUTE_MEMORY_MB=8192
+      # Default 300000 (5 min) per OmniRoute's own docs, but this deployment
+      # had it dialed down elsewhere (dashboard) to ~95s — too tight for a
+      # contended local llama-server: large-context prefill under multiple
+      # concurrent slots can outrun that before the first SSE token arrives,
+      # so OmniRoute cancels a request that was actually still working (see
+      # LLAMA_PARALLEL above for the other half of this fix). Raised here so
+      # it's tracked in git instead of a dashboard-only setting.
+      - STREAM_IDLE_TIMEOUT_MS=${OMNIROUTE_STREAM_IDLE_TIMEOUT_MS:-180000}
+    # Same reasoning as litellm's extra_hosts entry below — ai-stack's bridge
+    # network can't resolve search.home on its own.
    extra_hosts:
      - "search.home:${SEARXNG_LAN_IP}"
-    command: ["--config", "/app/config.yaml", "--port", "4000"]
    ports:
-      # published for LAN access (proxy.ai.home) and, via NPM, proxy.ai.haylan.ch —
-      # NPM must deny the /ui path on the external host. See docs/network-access.md.
-      - "${LITELLM_PORT:-4000}:4000"
+      - "${OMNIROUTE_API_PORT:-20129}:${OMNIROUTE_API_PORT:-20129}"
+      - "${OMNIROUTE_DASHBOARD_PORT:-20128}:${OMNIROUTE_DASHBOARD_PORT:-20128}"
+    # 10+ GiB ceiling per OmniRoute's Docker guide, matching
+    # OMNIROUTE_MEMORY_MB=8192 above.
+    mem_limit: 10g
+    # SQLite WAL needs time to checkpoint back into the main DB file on
+    # shutdown — the Docker guide's --stop-timeout 40 equivalent.
+    stop_grace_period: 40s
    restart: unless-stopped
    networks: [ai-stack]
+    # ponytail: TCP-connect check, not an HTTP /healthz GET — the image has
+    # no python3/curl/wget (confirmed live, `which` found only node), and
+    # OmniRoute's own Docker guide already treats a bare TCP probe on this
+    # port as an acceptable liveness check, not just the HTTP one. Simpler
+    # and avoids depending on /healthz's exact path/response shape.
    healthcheck:
      test:
        - CMD-SHELL
-        - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"
+        - node -e "require('net').connect(${OMNIROUTE_API_PORT:-20129},'localhost').on('connect',function(){this.end();process.exit(0)}).on('error',()=>process.exit(1))"
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

-  litellm-db:
-    image: postgres:16-alpine
-    container_name: litellm-db
-    env_file: .env
-    environment:
-      - POSTGRES_USER=litellm
-      - POSTGRES_PASSWORD=${LITELLM_DB_PASSWORD}
-      - POSTGRES_DB=litellm
-    volumes:
-      - litellm-db-data:/var/lib/postgresql/data
-    restart: unless-stopped
-    networks: [ai-stack]
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -d litellm -U litellm"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
-
-  # Backs litellm's router state, rate limits/budgets, and cache
-  # invalidation (see the litellm service's REDIS_* env vars above).
-  # ponytail: no persistence volume — everything litellm stores here is
-  # cache/coordination state it's fine to lose on restart, not source data.
-  redis:
-    image: redis:7-alpine
-    container_name: redis
-    command: ["redis-server", "--requirepass", "${REDIS_PASSWORD}"]
-    restart: unless-stopped
-    networks: [ai-stack]
-    healthcheck:
-      test: ["CMD-SHELL", "redis-cli -a ${REDIS_PASSWORD} ping | grep -q PONG"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
-
-  # Separate Postgres instance (with the pgvector extension) for the
-  # knowledgebase — NOT the same database as litellm-db, which is plain
-  # postgres:16-alpine and has no vector extension installed. See
-  # docs/research/litellm-knowledgebase.md.
-  pgvector-db:
-    image: pgvector/pgvector:pg16
-    container_name: pgvector-db
-    env_file: .env
-    environment:
-      - POSTGRES_USER=litellm_pgvector
-      - POSTGRES_PASSWORD=${PGVECTOR_DB_PASSWORD}
-      - POSTGRES_DB=litellm_pgvector
-    volumes:
-      - pgvector-db-data:/var/lib/postgresql/data
-    restart: unless-stopped
-    networks: [ai-stack]
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -d litellm_pgvector -U litellm_pgvector"]
-      interval: 5s
-      timeout: 5s
-      retries: 10
-
-  # LiteLLM's native knowledgebase/vector-store feature has no Qdrant backend
-  # (the qdrant service above only serves Open WebUI's own RAG/Memory) — this
-  # companion service (github.com/BerriAI/litellm-pgvector) is the only
-  # self-hosted path. No published image exists yet, so this builds from a
-  # vendored copy in vendor/litellm-pgvector/ (see that dir's README) rather
-  # than a remote git build context — the server's Docker/BuildKit couldn't
-  # do an authenticated-looking clone of a public github.com repo (fails
-  # with "could not read Username ... terminal prompts disabled"), and
-  # vendoring sidesteps needing that debugged. See
-  # docs/research/litellm-knowledgebase.md.
-  # ponytail: unverified against real hardware — Prisma migration behavior on
-  # first boot and the exact vector_store_registry field names for the
-  # pg_vector provider need a live smoke test. See issue #24.
-  litellm-pgvector:
-    build:
-      context: ./vendor/litellm-pgvector
-    container_name: litellm-pgvector
-    depends_on:
-      pgvector-db:
-        condition: service_healthy
-      litellm:
-        condition: service_healthy
-    environment:
-      - DATABASE_URL=postgresql://litellm_pgvector:${PGVECTOR_DB_PASSWORD}@pgvector-db:5432/litellm_pgvector
-      - SERVER_API_KEY=${LITELLM_PGVECTOR_API_KEY}
-      # Calls back into litellm for embeddings, same pattern as any other
-      # workload — see docs/proxy-key-onboarding.md for issuing this key.
-      # openai/ prefix required — litellm.aembedding can't infer a provider
-      # from a bare model name plus a custom api_base (raises "LLM Provider
-      # NOT provided"), same reasoning as the openai/ prefix on
-      # qwen3.8-27b-local and local-embedding in litellm-config.yaml.
-      - EMBEDDING__MODEL=openai/local-embedding
-      - EMBEDDING__BASE_URL=http://litellm:4000
-      - EMBEDDING__API_KEY=${LITELLM_PGVECTOR_EMBEDDING_KEY}
-      - EMBEDDING__DIMENSIONS=768
-    expose:
-      - "8000"
-    restart: unless-stopped
-    networks: [ai-stack]
-
  lazytainer:
    image: ghcr.io/vmorganp/lazytainer:master
    container_name: lazytainer
@@ -304,12 +303,40 @@ services:
    depends_on:
      - llama-server

+  # RAG vector store — see docs/agents/... (wayfinder). Dashboard UI published
+  # directly like comfyui above, not gatewayed through omniroute (it isn't an
+  # LLM provider).
+  qdrant:
+    image: qdrant/qdrant:latest
+    container_name: qdrant
+    volumes:
+      - qdrant-data:/qdrant/storage
+    ports:
+      - "6333:6333"
+    restart: unless-stopped
+    networks: [ai-stack]
+
+  # RAG graph store, native vector index too (can absorb qdrant's job later
+  # if the two-DB split proves unnecessary — see wayfinder notes).
+  neo4j:
+    image: neo4j:5-community
+    container_name: neo4j
+    environment:
+      - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD:?run scripts/update.sh first to resolve this}
+    volumes:
+      - neo4j-data:/data
+    ports:
+      - "7474:7474" # browser UI
+      - "7687:7687" # bolt
+    restart: unless-stopped
+    networks: [ai-stack]
+
 networks:
  ai-stack:

 volumes:
  models:
+  omniroute-data:
+  comfyui-data:
  qdrant-data:
-  openwebui-data:
-  litellm-db-data:
-  pgvector-db-data:
+  neo4j-data:
@@ -9,7 +9,7 @@ Use the **`tea` CLI** (already installed and authenticated as `haylan` via `tea
 - **Create an issue**: `tea issues create --title "..." --description "..." --labels "..."`
 - **Read an issue**: `tea issues <index> --comments`
 - **List issues**: `tea issues list --state open --labels "..."` (add `-f` to control which fields print)
- **Comment on an issue**: `tea comments create <index> --description "..."` (check `tea comments -h` for exact flags)
+- **Comment on an issue**: `tea comment <index> -d "..."` (check `tea comments -h` for exact flags — `tea comments create` is invalid, `add`/`a` is the subcommand)
 - **Apply / remove labels**: `tea issues edit <index> --add-labels "..."` / `--remove-labels "..."`
 - **Close**: `tea issues close <index>`
 - **Labels**: `tea labels create --name "..." --color "#hex" --description "..."`; `tea labels list`
@@ -48,4 +48,4 @@ Used by `/wayfinder`. This Gitea instance (1.27.2) has **no native sub-issue/par
 - **Blocking**: native issue dependencies via the raw API calls above. A ticket is unblocked when every dependency (`GET .../dependencies`) is closed.
 - **Frontier query**: `tea issues list --state open --labels "wayfinder:<type1>,wayfinder:<type2>,..."` scoped to the map's children (cross-check against the map's task list), drop any with an open dependency or an assignee.
 - **Claim**: `tea issues edit <n> --add-assignees haylan` — the session's first write.
- **Resolve**: `tea comments create <n> --description "<answer>"`, then `tea issues close <n>`, then append a context pointer (gist + link) to the map's Decisions-so-far, and check off its line in the map's task list.
+- **Resolve**: `tea comment <n> -d "<answer>"`, then `tea issues close <n>`, then append a context pointer (gist + link) to the map's Decisions-so-far, and check off its line in the map's task list. Map edits are full-body replaces (`tea issues edit` has no append) — concurrent resolutions racing on the same map issue can clobber each other's Decisions-so-far lines; re-fetch the map immediately before editing it, not from an earlier read.
@@ -1,91 +0,0 @@
-# Pointing a coding-agent CLI at this stack
-
-This stack routes through the [AI proxy](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) (LiteLLM) rather than talking to llama.cpp directly — llama.cpp's own port is internal-only now (see `docker-compose.yml`). The proxy exposes:
-
- **OpenAI-compatible**: `http://<ai-box>:4000/v1` (or `${LITELLM_PORT}` if you changed it in `.env`)
- **Anthropic Messages API** (LiteLLM's own unified `/v1/messages` endpoint, translating to the OpenAI-compatible backend): `http://<ai-box>:4000`
-
-Both serve the same underlying model — `Qwen3.8-27B-UD-Q4_K_XL.gguf`, registered in the proxy as `qwen3.8-27b-local` — behind whichever wire format the client speaks.
-
-`<ai-box>` is this machine's LAN address, or `proxy.ai.home` if your local DNS resolves that hostname directly to the box — see `docs/network-access.md`. If you're running a coding CLI from this machine itself, `localhost` works too.
-
-**Each CLI needs its own virtual key** — create one per docs/proxy-key-onboarding.md (LiteLLM's Admin UI, `<workload>-<purpose>` naming, e.g. `claude-code-cli`, `kimi-cli`, `opencode-cli`). No budget set by default. These are the machine's interactive/high-priority workloads per `docs/proxy-request-priority.md`.
-
-> **Read this before relying on it for real work.** Qwen3.8-27B's tool-calling has **documented, open llama.cpp upstream bugs** (parser fails on text before `<tool_call>`, tool calls emitted as inert XML inside thinking blocks — see `docs/research/qwen3.8-27b-tool-calling.md`). Every setup below inherits this risk identically, regardless of which CLI or wire format you use. Don't trust it for unattended multi-step agentic work until you've run the smoke test in [issue #5](https://git.arthurerlich.de/haylan/LLM-Server/issues/5) (and the proxy-specific smoke test in [issue #17](https://git.arthurerlich.de/haylan/LLM-Server/issues/17)).
-
-## Claude Code CLI
-
-Claude Code speaks the **Anthropic Messages API** — point it at the proxy's unified endpoint, not llama.cpp directly:
-
-```bash
-export ANTHROPIC_BASE_URL=http://<ai-box>:4000
-export ANTHROPIC_API_KEY=<claude-code-cli virtual key>
-claude
-```
-
-Requires llama.cpp's `--jinja` flag (already set in `docker-compose.yml`) — without it, tool-use requests fail outright.
-
-## Kimi CLI
-
-Kimi CLI speaks plain **OpenAI Chat Completions**. Configure a provider block in its config file (`config.toml`):
-
-```toml
-[providers.openai]
-type = "openai"
-base_url = "http://<ai-box>:4000/v1"
-api_key = "<kimi-cli virtual key>"
-```
-
-If Kimi CLI's response parsing gets confused by Qwen's `<think>...</think>` reasoning tags, check its `reasoning_key` setting — it's configurable for non-standard local server responses.
-
-## OpenCode CLI
-
-Confirmed project: **`anomalyco/opencode`** (renamed from `sst/opencode` — don't confuse with the unrelated `opencode-ai/opencode` Go TUI). Docs: https://opencode.ai/docs/
-
-**Install**:
-```bash
-curl -fsSL https://opencode.ai/install | bash
-```
-
-**Config** (`opencode.json`, project root or `~/.config/opencode/opencode.json`):
-
-```json
-{
-  "$schema": "https://opencode.ai/config.json",
-  "provider": {
-    "aiproxy": {
-      "npm": "@ai-sdk/openai-compatible",
-      "name": "AI proxy (local)",
-      "options": {
-        "baseURL": "http://<ai-box>:4000/v1",
-        "apiKey": "<opencode-cli virtual key>"
-      },
-      "models": {
-        "qwen3.8-27b-local": {
-          "name": "Qwen3.8-27B",
-          "limit": { "context": 65536, "output": 8192 }
-        }
-      }
-    }
-  }
-}
-```
-
-Set `limit.context` to match whatever `LLAMA_CTX_SIZE` this stack is actually running with (`.env`), not a value assumed from the model card — OpenCode uses it for its own context-management bookkeeping, not the server.
-
-Select the model with `aiproxy/qwen3.8-27b-local`.
-
-**OpenCode-specific risks** (on top of the shared Qwen3.8-27B tool-calling risk above):
- Requires llama.cpp's `--jinja` flag (already set) — without it, OpenCode's unconditional tool-call scaffolding gets a 500.
- [anomalyco/opencode#20669](https://github.com/anomalyco/opencode/issues/20669) (closed as "not planned" — a live, unfixed risk): OpenCode's `bash` tool crashes if the model omits the optional `description` field on a tool call; some local backends return `finish_reason: tool_calls` with an empty array, which can hang the agent loop instead of stopping cleanly.
- Thinking-mode handling (`options.reasoningEffort`) is undocumented for models that emit inline `<think>` tags rather than a native reasoning API field — expect no effect from that config on this model; untested.
-
-## Summary
-
-| CLI | Wire format | Endpoint | Config |
-|---|---|---|---|
-| Claude Code | Anthropic Messages | `http://<ai-box>:4000` | `ANTHROPIC_BASE_URL` env var |
-| Kimi CLI | OpenAI Chat Completions | `http://<ai-box>:4000/v1` | `config.toml` provider block |
-| OpenCode | OpenAI Chat Completions | `http://<ai-box>:4000/v1` | `opencode.json` provider block |
-
-Further reading: `docs/research/qwen3.8-27b-tool-calling.md`, `docs/research/opencode-cli-setup.md`, `docs/proxy-key-onboarding.md`.
@@ -0,0 +1,13 @@
+# Claude Code CLI
+
+[← back to overview](index.md)
+
+Claude Code speaks the **Anthropic Messages API** — point it at the gateway's unified endpoint, not llama.cpp directly:
+
+```bash
+export ANTHROPIC_BASE_URL=http://<ai-box>:${OMNIROUTE_PORT:-4000}
+export ANTHROPIC_API_KEY=<claude-code-cli virtual key>
+claude
+```
+
+Requires llama.cpp's `--jinja` flag (already set in `docker-compose.yml`) — without it, tool-use requests fail outright.
@@ -0,0 +1,34 @@
+# Pointing a coding-agent CLI at this stack
+
+[← back to README](../../README.md)
+
+This stack routes through the [AI gateway](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) (OmniRoute, see issue #31) rather than talking to llama.cpp directly — llama.cpp's own port is internal-only now (see `docker-compose.yml`). The gateway exposes:
+
+- **OpenAI-compatible**: `http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1`
+- **Anthropic Messages API** (OmniRoute's own `/v1/messages` endpoint, translating to the OpenAI-compatible backend): `http://<ai-box>:${OMNIROUTE_PORT:-4000}`
+
+Both serve the same underlying model — `Qwen3.8-27B-UD-Q4_K_XL.gguf`, registered in the gateway (naming is yours to pick when adding the llama-cpp provider connection — these docs assume `qwen3.8-27b-local` for continuity) — behind whichever wire format the client speaks.
+
+`<ai-box>` is this machine's LAN address, or `proxy-ai.home` if your local DNS resolves that hostname directly to the box — see `docs/network-access.md`. If you're running a coding CLI from this machine itself, `localhost` works too.
+
+**Each CLI needs its own virtual key** — create one per docs/proxy-key-onboarding.md (omniroute's dashboard, `<workload>-<purpose>` naming, e.g. `claude-code-cli`, `kimi-cli`, `opencode-cli`). No budget set by default. These are the machine's interactive/high-priority workloads per `docs/proxy-request-priority.md`.
+
+> **Read this before relying on it for real work.** Qwen3.8-27B's tool-calling has **documented, open llama.cpp upstream bugs** (parser fails on text before `<tool_call>`, tool calls emitted as inert XML inside thinking blocks — see `docs/research/qwen3.8-27b-tool-calling.md`). Every CLI below inherits this risk identically, regardless of wire format. Don't trust it for unattended multi-step agentic work until you've run the smoke test in [issue #5](https://git.arthurerlich.de/haylan/LLM-Server/issues/5) (and the proxy-specific smoke test in [issue #17](https://git.arthurerlich.de/haylan/LLM-Server/issues/17)).
+
+## Per-CLI setup
+
+- [Claude Code CLI](claude-code.md)
+- [Kimi CLI](kimi-cli.md)
+- [OpenCode CLI](opencode.md)
+- [Qwen Code CLI](qwen-code.md)
+
+## Summary
+
+| CLI | Wire format | Endpoint | Config |
+|---|---|---|---|
+| [Claude Code](claude-code.md) | Anthropic Messages | `http://<ai-box>:${OMNIROUTE_PORT:-4000}` | `ANTHROPIC_BASE_URL` env var |
+| [Kimi CLI](kimi-cli.md) | OpenAI Chat Completions | `http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1` | `config.toml` provider block |
+| [OpenCode](opencode.md) | OpenAI Chat Completions | `http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1` | `opencode.json` provider block |
+| [Qwen Code](qwen-code.md) | OpenAI Chat Completions (2 models: chat + `fastModel`) | `http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1` | `~/.qwen/settings.json` `modelProviders.openai` |
+
+Further reading: `docs/research/qwen3.8-27b-tool-calling.md`, `docs/proxy-key-onboarding.md`.
@@ -0,0 +1,14 @@
+# Kimi CLI
+
+[← back to overview](index.md)
+
+Kimi CLI speaks plain **OpenAI Chat Completions**. Configure a provider block in its config file (`config.toml`):
+
+```toml
+[providers.openai]
+type = "openai"
+base_url = "http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1"
+api_key = "<kimi-cli virtual key>"
+```
+
+If Kimi CLI's response parsing gets confused by Qwen's `<think>...</think>` reasoning tags, check its `reasoning_key` setting — it's configurable for non-standard local server responses.
@@ -0,0 +1,45 @@
+# OpenCode CLI
+
+[← back to overview](index.md)
+
+Confirmed project: **`anomalyco/opencode`** (renamed from `sst/opencode` — don't confuse with the unrelated `opencode-ai/opencode` Go TUI). Docs: https://opencode.ai/docs/
+
+**Install**:
+```bash
+curl -fsSL https://opencode.ai/install | bash
+```
+
+**Config** (`opencode.json`, project root or `~/.config/opencode/opencode.json`):
+
+```json
+{
+  "$schema": "https://opencode.ai/config.json",
+  "provider": {
+    "aiproxy": {
+      "npm": "@ai-sdk/openai-compatible",
+      "name": "AI proxy (local)",
+      "options": {
+        "baseURL": "http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1",
+        "apiKey": "<opencode-cli virtual key>"
+      },
+      "models": {
+        "qwen3.8-27b-local": {
+          "name": "Qwen3.8-27B",
+          "limit": { "context": 65536, "output": 8192 }
+        }
+      }
+    }
+  }
+}
+```
+
+Set `limit.context` to the *per-slot* context this stack actually serves — `LLAMA_CTX_SIZE / LLAMA_PARALLEL` from `.env` (262144 / 2 = 131072 by default), not raw `LLAMA_CTX_SIZE` and not a value assumed from the model card: llama.cpp divides `--ctx-size` across concurrent slots, so each request only gets one slot's share. OpenCode uses this for its own context-management bookkeeping, not the server.
+
+Select the model with `aiproxy/qwen3.8-27b-local`.
+
+**OpenCode-specific risks** (on top of the shared Qwen3.8-27B tool-calling risk — see [overview](index.md)):
+- Requires llama.cpp's `--jinja` flag (already set) — without it, OpenCode's unconditional tool-call scaffolding gets a 500.
+- [anomalyco/opencode#20669](https://github.com/anomalyco/opencode/issues/20669) (closed as "not planned" — a live, unfixed risk): OpenCode's `bash` tool crashes if the model omits the optional `description` field on a tool call; some local backends return `finish_reason: tool_calls` with an empty array, which can hang the agent loop instead of stopping cleanly.
+- Thinking-mode handling (`options.reasoningEffort`) is undocumented for models that emit inline `<think>` tags rather than a native reasoning API field — expect no effect from that config on this model; untested.
+
+Further reading: `docs/research/opencode-cli-setup.md`.
@@ -0,0 +1,116 @@
+# Qwen Code CLI
+
+[← back to overview](index.md)
+
+Qwen Code speaks plain **OpenAI Chat Completions**, and — unlike the other CLIs — needs *two* models: the main chat model, and a `fastModel` for Auto Mode's action classifier (a separate, always-resident, always-fast instance so classification doesn't queue behind chat prefill; see `docker-compose.yml`'s `llama-server-fast` service and `docs/research/fast-model-choice.md`). Both are registered as separate providers in OmniRoute but reachable through the same gateway URL. Config lives in `~/.qwen/settings.json`:
+
+```json
+{
+  "modelProviders": {
+    "openai": [
+      {
+        "id": "<main-model-provider-id-in-omniroute>",
+        "name": "qwen3.8-27b-local",
+        "envKey": "OMNIROUTE_API_KEY",
+        "baseUrl": "http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1",
+        "generationConfig": { "contextWindowSize": 131072 }
+      },
+      {
+        "id": "<fast-model-provider-id-in-omniroute>",
+        "name": "qwen3.8-27b-classifier",
+        "envKey": "OMNIROUTE_API_KEY",
+        "baseUrl": "http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1",
+        "generationConfig": {
+          "contextWindowSize": 8192,
+          "extra_body": { "chat_template_kwargs": { "enable_thinking": false } }
+        }
+      }
+    ]
+  },
+  "security": { "auth": { "selectedType": "openai" } },
+  "model": {
+    "name": "<main-model-provider-id-in-omniroute>",
+    "baseUrl": "http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1"
+  },
+  "fastModel": "<fast-model-provider-id-in-omniroute>"
+}
+```
+
+- `envKey` names the environment variable Qwen Code reads the virtual key from — set `OMNIROUTE_API_KEY=<qwen-code-cli virtual key>` before launching. Both providers can share one virtual key (as above); split it into two if you want separate usage tracking for chat vs. classifier calls.
+- **`contextWindowSize` is per-slot, not `LLAMA_CTX_SIZE` itself** — llama.cpp divides `--ctx-size` across `LLAMA_PARALLEL` concurrent slots, and each request only gets one slot's share (same correction applies to OpenCode's `limit.context`). Compute it per model from `.env`:
+  - Main model: `LLAMA_CTX_SIZE / LLAMA_PARALLEL` = `262144 / 2` = **131072**.
+  - Fast model: `LLAMA_FAST_CTX_SIZE / LLAMA_FAST_PARALLEL` = `8192 / 1` = **8192**. Undersizing this one specifically breaks Auto Mode ("Classifier stage 1 unavailable") once `hints.allow`/`softDeny`/`hardDeny` entries and recent-action history push a classifier call past it — see the `LLAMA_FAST_CTX_SIZE` comment in `.env.example` before raising it instead of `LLAMA_FAST_PARALLEL`.
+- `enable_thinking: false` on the fast model matters: the fast model file (`Qwen3-4B-Instruct-2507`) is already non-thinking, but this also suppresses `<think>` output on any fast-model swap that isn't, keeping classifier responses parseable.
+- Qwen Code also recognizes `advisorModel`, `visionModel`, `compactionModel`, `imageModel` for other model roles — none are wired up in this stack; only `fastModel` is required.
+
+## Web search via OmniRoute
+
+Qwen Code's own built-in web search (`tools.webSearch.enabled`) has nothing to search with here — leave it `false`. Instead this stack's SearXNG-backed search (README §"Web search") is exposed through a thin stdio MCP wrapper around OmniRoute's `/v1/search` REST endpoint (that endpoint isn't itself MCP — OmniRoute's real MCP surface is admin-only/LOCAL_ONLY-gated). Save this as e.g. `~/.qwen/mcp-servers/omniroute-search/index.mjs` (needs `@modelcontextprotocol/sdk` and `zod`: `npm init -y && npm i @modelcontextprotocol/sdk zod` in that directory):
+
+```js
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { z } from "zod";
+
+const BASE_URL = process.env.OMNIROUTE_BASE_URL || "http://proxy-ai.home";
+const API_KEY = process.env.OMNIROUTE_API_KEY;
+
+if (!API_KEY) {
+  console.error("OMNIROUTE_API_KEY is not set in the environment.");
+  process.exit(1);
+}
+
+const server = new McpServer({ name: "omniroute-search", version: "1.0.0" });
+
+server.registerTool(
+  "search",
+  {
+    description: "Web/news search via OmniRoute's /v1/search endpoint.",
+    inputSchema: { query: z.string().describe("Search query") },
+  },
+  async ({ query }) => {
+    const res = await fetch(`${BASE_URL}/v1/search`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", Authorization: `Bearer ${API_KEY}` },
+      body: JSON.stringify({ query }),
+    });
+    const text = await res.text();
+    if (!res.ok) return { content: [{ type: "text", text: `HTTP ${res.status}: ${text}` }], isError: true };
+    return { content: [{ type: "text", text }] };
+  }
+);
+
+await server.connect(new StdioServerTransport());
+```
+
+Register it in `~/.qwen/settings.json`:
+
+```json
+{
+  "mcpServers": {
+    "omniroute-search": { "command": "node", "args": ["<path-to>/index.mjs"] }
+  },
+  "tools": { "webSearch": { "enabled": false } }
+}
+```
+
+It reuses the same `OMNIROUTE_API_KEY` env var as the model providers above — the virtual key needs search permission in OmniRoute, not just chat-completions.
+
+## Auto Mode tuning
+
+Auto Mode's action classifier calls the fast model above — its own request can queue behind other stack traffic before the fast llama-server instance is warm, so the default classifier timeout is worth raising. And since this stack is a single trusted local proxy, it's reasonable to pre-approve requests to it rather than confirm every call:
+
+```json
+{
+  "permissions": {
+    "autoMode": {
+      "classifier": { "timeouts": { "stage1Ms": 600000 } },
+      "hints": { "allow": ["Requests to proxy-ai.home, my own local omniroute model proxy"] }
+    }
+  }
+}
+```
+
+`hints.allow` entries are free-text descriptions the classifier matches against, not exact strings — capped at 150 entries/200 chars each (see the `LLAMA_FAST_CTX_SIZE` note above for why that ceiling matters).
+
+Everything else in `~/.qwen/settings.json` (`hooks`, `security.auth`'s underlying tooling, editor prefs) is per-machine, not part of pointing at this stack — don't copy it wholesale between machines.
@@ -1,66 +0,0 @@
-# Knowledgebase, memory, and web search
-
-Three gateway-level capabilities added on top of the [AI gateway/proxy](https://git.arthurerlich.de/haylan/LLM-Server/issues/9), so every client behind LiteLLM gets them — not just Open WebUI. See [issue #21](https://git.arthurerlich.de/haylan/LLM-Server/issues/21) for the rationale.
-
-**Verified against a live deploy** — see [issue #24](https://git.arthurerlich.de/haylan/LLM-Server/issues/24), closed after smoke-testing found and fixed several bugs: a missing `api_key` in `vector_store_registry` (was silently falling through to the real `api.openai.com`), `litellm-pgvector`'s Prisma schema never actually being pushed to `pgvector-db` (now handled by `./scripts/update.sh`), a 1536- vs 768-dim vector column mismatch, and its create endpoint ignoring any caller-supplied store id (both fixed locally — see `vendor/litellm-pgvector/VENDORED.md`). `scripts/ingest-memory.sh` was also silently broken (posted chunks with no embedding attached) and has been fixed to embed via LiteLLM before inserting.
-
-## Web search (SearXNG)
-
-`litellm-config.yaml`'s `search_tools` block wires the LAN's SearXNG instance in as a **standalone REST endpoint**, not a model-callable tool — call it directly:
-
-```bash
-curl http://<proxy>:4000/v1/search/searxng-search \
-  -H "Authorization: Bearer <a virtual key>" \
-  -H "Content-Type: application/json" \
-  -d '{"query": "...", "max_results": 5}'
-```
-
-Because this doesn't ask the model to emit a tool call, it sidesteps Qwen3.8-27B's known-flaky tool-calling (`docs/research/qwen3.8-27b-tool-calling.md`) entirely. Open WebUI's own web-search setting can point at this endpoint the same way.
-
-Requires `SEARXNG_LAN_IP` set in `.env` so the `litellm` container can resolve `search.home` via `extra_hosts` — `./scripts/update.sh` resolves and fills this in automatically from the host's own DNS if it's blank (use a static DHCP reservation for `search.home` so it doesn't drift). Full research: `docs/research/litellm-searxng-search.md`.
-
-## Knowledgebase (vector store / RAG)
-
-LiteLLM's native knowledgebase feature has **no Qdrant backend** — the `qdrant` service in this stack only serves Open WebUI's own separate RAG/Memory feature and is unrelated to this. The only self-hosted path is [litellm-pgvector](https://github.com/BerriAI/litellm-pgvector), a companion service backed by its own Postgres+pgvector database (`pgvector-db`), which this stack now runs alongside `litellm`. Full research: `docs/research/litellm-knowledgebase.md`.
-
-New pieces:
-
- **`embedding-server`** — a second llama.cpp instance (small footprint, `nomic-embed-text-v1.5`) serving `/v1/embeddings`. The chat model isn't embedding-trained and llama.cpp serves one model per process, so this can't just be a flag on `llama-server`.
- **`pgvector-db`** — Postgres with the pgvector extension, separate from `litellm-db`.
- **`litellm-pgvector`** — the connector service; no published image exists, so it's built from a vendored copy of the upstream repo at `vendor/litellm-pgvector/` (see that dir's `VENDORED.md`) — a remote git build context failed on the server's Docker/BuildKit setup.
- `litellm-config.yaml`'s `local-embedding` model entry and `vector_store_registry` block, tying it together.
-
-### First-time setup
-
-`./scripts/update.sh` fetches the embedding model automatically (skips it if already downloaded). To do it by hand instead:
-
-```bash
-docker compose --profile tools run --rm downloader-embedding   # fetch the embedding model
-docker compose up -d embedding-server pgvector-db litellm-pgvector
-```
-
-`./scripts/update.sh` mints `LITELLM_PGVECTOR_EMBEDDING_KEY` automatically (a `litellm-pgvector` virtual key via LiteLLM's own API) if it's blank — it calls back into `litellm` for embeddings, same as any other workload. See `docs/proxy-key-onboarding.md` if a mint fails and it needs doing by hand.
-
-### Loading memory into it
-
-`data/memory.md` and `data/claude-legacy-memory.md` — Claude-memory-style fact files — get loaded via:
-
-```bash
-./scripts/ingest-memory.sh
-```
-
-One chunk per fact/paragraph line, tagged with `source`/`section` metadata. Re-run after editing either file (see the script's header comment for the no-dedup caveat).
-
-### Querying it
-
-Via the OpenAI Assistants-style `file_search` tool on a chat completion:
-
-```json
-{
-  "model": "qwen3.8-27b-local",
-  "messages": [...],
-  "tools": [{"type": "file_search", "vector_store_ids": ["memory-and-notes"]}]
-}
-```
-
-or directly: `POST /v1/vector_stores/memory-and-notes/search` with `{"query": "..."}`.
@@ -1,27 +1,24 @@
-# Network access: ai.home / ai.haylan.ch
+# Network access: proxy-ai.home / proxy-ai.haylan.ch

-Open WebUI is meant to be reachable as **`ai.home`** on the LAN and **`ai.haylan.ch`** from outside. This stack doesn't run its own reverse proxy — it publishes Open WebUI's port to the host (`${WEBUI_PORT:-3000}`, see `docker-compose.yml`) and relies on the **existing Nginx Proxy Manager (NPM)** instance already fronting other self-hosted services on this network.
-
-## What to set up in NPM
-
-Two Proxy Hosts, both pointing at this machine's LAN IP on port `${WEBUI_PORT:-3000}` (Open WebUI):
-
- **`ai.home`** — internal only, no external DNS/TLS needed unless you want it.
- **`ai.haylan.ch`** — external, reachable via the DMZ already forwarding it to NPM; let NPM issue/manage the TLS cert as it does for other services.
+This stack has no chat UI — every client is a coding CLI reaching the AI gateway (OmniRoute). It doesn't run its own reverse proxy — it publishes the gateway's API port to the host and relies on the **existing Nginx Proxy Manager (NPM)** instance already fronting other self-hosted services on this network.

 ## llama.cpp's raw API stays LAN-only — deliberately

-The inference API (port `${LLAMA_PORT:-8080}`) is **not** registered in NPM and is **not** reachable via `ai.haylan.ch`. It has no authentication of its own (unlike Open WebUI, which has login enabled) — putting it on the public internet would mean an unauthenticated inference endpoint. Coding-agent CLIs (Claude Code, Kimi, OpenCode — see `docs/coding-cli-setup.md`) reach it directly over the LAN, using this machine's LAN IP or `ai.home` if your local DNS resolves that hostname straight to the box (bypassing NPM, which only fronts ports 80/443).
+The inference API (port `${LLAMA_PORT:-8080}`) is **not** registered in NPM and is **not** reachable externally. It has no authentication of its own — putting it on the public internet would mean an unauthenticated inference endpoint. Coding-agent CLIs (Claude Code, Kimi, OpenCode, Qwen Code — see `docs/coding-cli-setup/`) don't reach it directly at all now; they go through the gateway below, same as everything else.

-If you later want external CLI access too, that's a deliberate scope change — see the map ([issue #1](https://git.arthurerlich.de/haylan/LLM-Server/issues/1)) before doing it, since it changes the security posture (the raw API would need its own auth in front of it).
+If you later want external CLI access too, that's a deliberate scope change — see the map ([issue #1](https://git.arthurerlich.de/haylan/LLM-Server/issues/1)) before doing it, since it changes the security posture.

-## The AI proxy (LiteLLM) — `proxy.ai.home` / `proxy.ai.haylan.ch`
+## The AI gateway (OmniRoute) — `proxy-ai.home` / `proxy-ai.haylan.ch`

-Once the gateway from [issue #9](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) is deployed, it gets its own hostnames, same NPM pattern as Open WebUI above:
+As of [issue #31](https://git.arthurerlich.de/haylan/LLM-Server/issues/31) (migrated from LiteLLM), the gateway is OmniRoute:

- **`proxy.ai.home`** — internal only, fronts the full LiteLLM port (API + Admin UI).
- **`proxy.ai.haylan.ch`** — external, via the DMZ/NPM. Fronts only the inference API paths.
+- **`proxy-ai.home`** and **`proxy-ai.haylan.ch`** both point only at `${OMNIROUTE_PORT:-4000}` — the API port. Set up as two NPM Proxy Hosts pointing at this machine's LAN IP on that port; `proxy-ai.home` internal-only, `proxy-ai.haylan.ch` external via the DMZ already forwarding to NPM (let NPM issue/manage the TLS cert as usual).
+- The **dashboard** (`${OMNIROUTE_DASHBOARD_PORT:-20128}`) is never registered in NPM at all, and `docker-compose.yml` never publishes that port to the host either — it manages every workload's keys, so it doesn't belong on the public internet, same reasoning as LiteLLM's old `/ui`. Unlike LiteLLM, OmniRoute's split-port mode means this is structural (no network route exists) rather than an NPM path-deny rule that has to be maintained and could be misconfigured. Reach the dashboard only from the host itself or over SSH port-forward.

-**Every proxy call already requires a valid virtual key** (Bearer token, see `docs/proxy-key-onboarding.md`) — the same bar Open WebUI clears with its own login — so no extra NPM-level auth is needed for the external hostname.
+**Every gateway call already requires a valid API key** (Bearer token, see `docs/proxy-key-onboarding.md`), so no extra NPM-level auth is needed for the external hostname.

-**LiteLLM's Admin UI (`/ui`) stays LAN-only**, same reasoning as llama.cpp's raw API: it manages every workload's keys and budgets, so it doesn't belong on the public internet. LiteLLM serves `/ui` on the same port as its API by default, so `proxy.ai.haylan.ch`'s NPM Proxy Host needs an explicit rule denying the `/ui` path (a "Deny" custom location, same UI as the "Advanced" tab used for other NPM hosts) — `proxy.ai.home` has no such restriction and reaches both the API and the Admin UI.
+## RAG knowledge graph (Neo4j) — `knowledge.proxy-ai.home`
+
+Set up as an NPM Proxy Host pointing at this machine's LAN IP on Neo4j's Browser port (`7474`, see `docker-compose.yml`'s `neo4j` service, [PR #50](https://git.arthurerlich.de/haylan/LLM-Server/pulls/50)). Internal-only, same as `proxy-ai.home` — no DMZ/external route, this is admin/dev tooling, not a client-facing endpoint. Bolt (`7687`, the actual query protocol) isn't proxied through NPM at all — clients on the LAN reach it directly at `<this-machine>:7687`.
+
+Qdrant's dashboard (`6333`) stays on its raw LAN IP/port for now — no hostname assigned yet.
@@ -1,25 +1,17 @@
-# Onboarding a workload onto the AI proxy
+# Onboarding a workload onto the AI gateway

-How to issue a new per-workload API key against the LiteLLM proxy (see [issue #10](https://git.arthurerlich.de/haylan/LLM-Server/issues/10) / `docs/research/proxy-tool-choice.md`), so a new workload (a code-reviewer tool, Paperless-OCR, Gitea code review, etc.) gets its own key and its own visible usage/spend.
+How to issue a new per-workload API key against the OmniRoute gateway (see [issue #31](https://git.arthurerlich.de/haylan/LLM-Server/issues/31) — the LiteLLM → OmniRoute migration; original gateway rationale in [issue #10](https://git.arthurerlich.de/haylan/LLM-Server/issues/10) / `docs/research/proxy-tool-choice.md`), so a new workload (a code-reviewer tool, Paperless-OCR, Gitea code review, etc.) gets its own key and its own visible usage/spend.

-`OPENWEBUI_LITELLM_KEY` and `LITELLM_PGVECTOR_EMBEDDING_KEY` — the two keys this stack's own services need — are minted automatically by `./scripts/update.sh` via the same API `curl` shows below; the steps here are for any other workload, or for those two if the automatic mint ever fails.
+No workload in this stack itself needs a key right now — every client is external (a coding CLI, or another self-hosted service). There's no scripted mint yet either way: `POST /api/keys` needs a dashboard login session (`ManagementSessionAuth`), not a static bearer key like LiteLLM's old `/key/generate`, and that flow hasn't been verified against a live instance (see [issue #37](https://git.arthurerlich.de/haylan/LLM-Server/issues/37)). Create every key by hand for now, via the dashboard steps below.

 ## Create the key

-1. Log into LiteLLM's Admin UI (`/ui` on the proxy's deployed URL).
-2. Create a new virtual key ("Keys" → "Create Key").
-3. Name it `<workload>-<purpose>` — a short slug matching the workload, e.g. `paperless-ocr`, `gitea-code-review`, `openwebui`. This name is the ledger: LiteLLM's dashboard lists keys by name, so there's no separate tracking doc to keep in sync — name it clearly and the Usage tab tells you the rest (spend, last used, etc.).
-4. Leave budget and rate limits unset (unlimited) by default. This is a shadow-cost estimate for fun, not real accounting or resource protection — see `docs/research/proxy-shadow-pricing.md`. Only set a budget if a specific workload turns out to need a tripwire.
+1. Log into the omniroute dashboard. `DASHBOARD_PORT` (20128) is never published to the host (see `docker-compose.yml`'s `omniroute` service) — from the R9700 box itself, find the container's own address (`docker inspect -f '{{.NetworkSettings.Networks.ai_stack.IPAddress}}' omniroute`) and browse to `http://<that-ip>:20128` (the host can reach a container's bridge-network IP directly, published port or not). From elsewhere, SSH port-forward instead: `ssh -L 20128:<container-ip>:20128 <host>`, then browse `http://localhost:20128`.
+2. "Keys" → "Create API key".
+3. Label it `<workload>-<purpose>` — a short slug matching the workload, e.g. `paperless-ocr`, `gitea-code-review`, `claude-code-cli`. This label is the ledger: the dashboard lists keys by label, so there's no separate tracking doc to keep in sync.
+4. Copy the key value shown — it's only shown once at creation, per OmniRoute's docs.

-Or the same thing over the API (what `update.sh` does):
-
-```bash
-curl -sf -X POST "http://<proxy>:4000/key/generate" \
-  -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \
-  -H "Content-Type: application/json" \
-  -d '{"key_alias": "<workload>-<purpose>"}'
-# -> {"key": "sk-...", ...}
-```
+Once `POST /api/keys`'s session-auth flow is worked out (issue #37), the equivalent `curl` here can replace this manual step, the way `update.sh` used to automate LiteLLM's `/key/generate`.

 ## Hand it to the workload

@@ -27,7 +19,7 @@ Drop the key into that workload's own `.env` (or equivalent config) — never in

 ## Retiring or rotating a key

-No scheduled rotation. Revoke the key by hand in the Admin UI ("Keys" → delete) only when:
+No scheduled rotation. Revoke the key by hand in the dashboard ("Keys" → delete) only when:
 - the workload is retired, or
 - the key is suspected leaked/compromised.

@@ -1,12 +1,14 @@
 # Request priority on the AI proxy

-One local model instance (llama.cpp on the single R9700) serves every workload through the LiteLLM proxy ([issue #9](https://git.arthurerlich.de/haylan/LLM-Server/issues/9)). Interactive usage shouldn't get stuck behind a batch job.
+**Stale as of [issue #31](https://git.arthurerlich.de/haylan/LLM-Server/issues/31) (LiteLLM → OmniRoute migration)** — the Mechanism section below describes LiteLLM's specific scheduler, which no longer applies. Whether OmniRoute has an equivalent priority/queueing mechanism hasn't been researched. The Tiers/problem statement below still holds; treat Mechanism onward as historical until this is revisited.
+
+One local model instance (llama.cpp on the single R9700) serves every workload through the AI gateway ([issue #9](https://git.arthurerlich.de/haylan/LLM-Server/issues/9)). Interactive usage shouldn't get stuck behind a batch job.

 ## Tiers

 Two tiers, assigned per workload's virtual key (per `docs/proxy-key-onboarding.md`):

- **High priority** (interactive — someone's waiting): Open WebUI chat, coding CLIs (Claude Code / Kimi / OpenCode), Gitea code review.
+- **High priority** (interactive — someone's waiting): coding CLIs (Claude Code / Kimi / OpenCode), Gitea code review.
 - **Low priority** (batch — nobody's watching a spinner): Paperless OCR/tagging, Nextcloud Memories face-recognition, AI watermark removal.

 ## Mechanism
@@ -0,0 +1,412 @@
+# Research: self-hosted alternatives to DashScope for Qwen Code's built-in `web_search` tool
+
+**Question:** Qwen Code CLI's built-in `web_search` tool requires `tools.webSearch.model`
+to resolve to a "DashScope-compatible `modelProviders` entry." Is there any real,
+non-Alibaba-Cloud way to satisfy that requirement with something self-hosted —
+or is the already-working OmniRoute MCP + SearXNG path (`docs/research/omniroute-qwen-websearch.md`)
+the end of the road?
+
+**Answer, short version:** No. The client-side code that decides whether a
+`baseUrl` is "DashScope-compatible" checks the **literal hostname** against a
+hardcoded allowlist of Alibaba-owned domains, before any request is sent — it
+is not a protocol-compatibility check that a look-alike server could pass. A
+self-hosted server cannot satisfy it, full stop, unless you fork qwen-code and
+delete that check. Once you've done that, the actual wire protocol
+(OpenAI SDK `responses.create()`, SSE, specific item types) is buildable
+(a few hundred lines), but nothing you can install off the shelf implements it
+today. The already-working OmniRoute MCP + SearXNG path costs nothing further
+and does not have this problem. **Recommendation: don't build this — see
+§6.**
+
+## 1. What "DashScope Responses API" is, precisely
+
+Alibaba Cloud Model Studio (Bailian/DashScope) added an **OpenAI-compatible
+Responses API**, layered on top of its existing Chat Completions
+compatible-mode surface:
+
+- Endpoint (per Alibaba's own docs): `POST {baseUrl}/responses`, where
+  `baseUrl` is the region's compatible-mode base, e.g.
+  `https://dashscope.aliyuncs.com/compatible-mode/v1` (China/Beijing) or the
+  `-intl` / regional `*.maas.aliyuncs.com` variants.
+  Source: https://www.alibabacloud.com/help/en/model-studio/qwen-api-via-openai-responses
+  and https://www.alibabacloud.com/help/en/model-studio/compatibility-with-openai-responses-api
+- Request shape: standard Responses API (`model`, `input`, `stream`, `store`,
+  `instructions`) plus a `tools` array that can include
+  `{"type": "web_search"}`, `{"type": "web_extractor"}`, `{"type": "code_interpreter"}`
+  as **hosted, server-side tools** — the inference backend runs the search
+  itself and streams results back, the same hosted-tool pattern as OpenAI's
+  own Responses API `web_search_preview`, not a client-side function-calling
+  round trip.
+  Source: https://www.alibabacloud.com/help/en/model-studio/qwen-api-via-openai-responses
+- Response shape: an `output` array of typed items — a `web_search_call` item
+  carries `action: {type, query, sources: [{type: "url", url}]}`; narration
+  comes back as `message` items with `content: [{type, text}]`.
+  Source: same page.
+- Separately, DashScope's plain Chat Completions endpoint (not Responses)
+  has an older, unrelated `enable_search` boolean (passed via `extra_body`)
+  for models like Qwen3.8/Qwen3.6-Plus — the docs explicitly note this older
+  surface **does not return citations/sources**, which is exactly why
+  qwen-code's built-in tool uses the *Responses* API instead.
+  Source: https://docs.qwencloud.com/developer-guides/tool-calling/web-search
+
+This is the same hosted-tool pattern as OpenAI's Responses API
+`web_search_preview` (§4 confirms this directly from qwen-code's own client
+code — it literally reuses the OpenAI Node SDK's `responses.create()` call
+against a DashScope base URL).
+
+## 2. What qwen-code's own client code actually sends (ground truth)
+
+Fetched directly from `QwenLM/qwen-code`'s `main` branch,
+`packages/core/src/tools/web-search.ts` (1087 lines) and
+`packages/core/src/core/openaiContentGenerator/{constants,provider/dashscope}.ts`.
+This supersedes anything inferred from the docs pages — it's the literal
+implementation.
+
+**The request** (`web-search.ts` lines 651–699):
+
+```ts
+const client = new OpenAI({
+  apiKey,                    // from the resolved modelProviders entry's envKey
+  baseURL: backend.baseUrl,  // the modelProviders entry's baseUrl / WEB_SEARCH_BASE_URL
+  timeout: 60_000,
+  defaultHeaders: { 'User-Agent': `QwenCode/${version} (...)`, ...customHeaders },
+});
+
+const tools = [{ type: 'web_search' }];
+if (backend.webExtractor) tools.push({ type: 'web_extractor' });
+
+const requestParams = {
+  model: backend.modelId,
+  input: `Perform a web search for the query: ${query}`,
+  stream: true,
+  store: false,
+  instructions: SIDE_REQUEST_INSTRUCTIONS,   // a fixed system prompt, see source
+  tools,
+};
+
+const stream = await client.responses.create(requestParams, { signal });
+```
+
+This is the **official OpenAI Node SDK**, so `client.responses.create()`
+literally POSTs to `{baseURL}/responses` with that JSON body and reads back
+an SSE stream — there is no DashScope-specific SDK involved at all. Anything
+speaking real OpenAI Responses-API SSE syntax at that path, with these two
+extra tool types, is protocol-compatible.
+
+**What the client parses out of the SSE stream** (lines 359–509): event types
+`response.output_item.added`, `response.output_item.done`,
+`response.output_text.delta`, and terminal `response.completed` /
+`.failed` / `.incomplete` / `.cancelled`, each carrying a `response` object
+with `output: WsOutputItem[]` and `usage.x_tools.{web_search,web_extractor}.count`.
+Output items it understands: `web_search_call` (`action.query`/`action.queries`,
+`action.sources[].url`, `status`), `web_extractor_call` (`urls`, `goal`,
+`output`, `status`), and `message` (`content[].text`). It also defensively
+handles a DashScope-specific quirk: **request-level failures arrive as a bare
+SSE `event:error` with `{code, message, request_id}` and no `type`/`error`
+wrapper** — the OpenAI SDK doesn't recognize this shape, so qwen-code parses
+it itself (comment: "probe-verified"). Any replacement backend needs to emit
+exactly these item/event shapes, or qwen-code's parser silently ignores
+unrecognized item types and ultimately reports
+`WEB_SEARCH_NO_SEARCH_PERFORMED` (it treats zero `web_search_call` items as
+"no search happened," with one retry, before failing outright — see lines
+883–906).
+
+**The hard gate — this is the actual finding.** Before any request is sent,
+`evaluateWebSearchGate()` (lines 169–335) validates the resolved `baseUrl`
+through `classifyDashScopeBaseUrl()` (lines 122–157):
+
+```ts
+function classifyDashScopeBaseUrl(baseUrl: string): DashScopeBaseUrlIssue | null {
+  const url = new URL(baseUrl);
+  if (url.protocol !== 'https:') return 'insecure';
+  const hostname = url.hostname.toLowerCase();
+  const suffixes = [
+    ...DASHSCOPE_REGIONAL_HOSTS,   // dashscope.aliyuncs.com, dashscope-intl.aliyuncs.com, dashscope-us.aliyuncs.com
+    'maas.aliyuncs.com',
+    'alibaba-inc.com',
+    'aliyun-inc.com',
+  ];
+  return suffixes.some(s => hostname === s || hostname.endsWith('.' + s)) ? null : 'unknown-host';
+}
+```
+
+`DASHSCOPE_REGIONAL_HOSTS` is defined in
+`packages/core/src/core/openaiContentGenerator/provider/dashscope.ts` as
+exactly `['dashscope.aliyuncs.com', 'dashscope-intl.aliyuncs.com', 'dashscope-us.aliyuncs.com']`.
+
+**This means "DashScope-compatible" is not a protocol claim you can satisfy
+by implementing the right JSON shapes — it is a literal hostname allowlist
+checked client-side, before the request is even built.** A self-hosted server
+at `http://search.home`, `https://proxy-ai.home`, or any hostname you control
+will be rejected with *"WebSearch ... is not a DashScope-compatible
+endpoint"* regardless of what protocol it speaks, unless its hostname ends in
+one of `dashscope.aliyuncs.com`, `dashscope-intl.aliyuncs.com`,
+`dashscope-us.aliyuncs.com`, `*.maas.aliyuncs.com`, `*.alibaba-inc.com`, or
+`*.aliyun-inc.com` — domains Alibaba owns, that you cannot obtain a valid TLS
+certificate for. (There's also a separate, unrelated `DASHSCOPE_PROXY_BASE_URL`
+env var used by the *main* content generator's provider-detection code
+(`dashscope.ts` lines 244–262) for header/cache-control routing through a
+corporate proxy — it is not consulted by `classifyDashScopeBaseUrl()` at all,
+so it does not help here either.)
+
+The only way around this specific check is to **fork qwen-code and delete or
+relax `classifyDashScopeBaseUrl()`** — it's ~15 lines of open-source
+TypeScript, so this is not hard *code-wise*, but it means running a patched
+build of the CLI, not configuring the stock release.
+
+## 3. Any self-hostable server implementing this surface today? — No
+
+Checked the servers this task named:
+
+- **vLLM**: has a real `/v1/responses` implementation
+  (https://docs.vllm.ai/en/stable/api/vllm/entrypoints/openai/responses/), and
+  for `gpt-oss` models specifically supports a **built-in `browser` tool**
+  with a pluggable, MCP-compliant external tool server in place of the
+  default Exa-backed reference implementation
+  (https://vllm.ai/blog/2025-08-05-gpt-oss;
+  https://github.com/vllm-project/recipes/blob/main/OpenAI/GPT-OSS.md). This
+  is the closest existing building block found — but it's gpt-oss/harmony
+  specific (not Qwen), and its tool/event shapes (`browser.search`,
+  `browser.open` harmony-channel messages) are **not** the same as DashScope's
+  `web_search_call`/`web_extractor_call` items qwen-code's parser expects, so
+  it is not drop-in — it would need a translation shim in front, at which
+  point you're building the shim anyway and don't need vLLM in the path.
+- **SGLang**: Responses API support is unclear/inconsistent per its own
+  issue tracker (https://github.com/sgl-project/sglang/issues/10038) — no
+  usable built-in web-search tool found.
+- **LiteLLM**: does expose `/v1/responses`, but has an **open bug**
+  rejecting the `web_search` tool type outright — "LiteLLM raises a
+  validation error... only `web_search_preview` is currently allowed"
+  (https://github.com/BerriAI/litellm/issues/14011). Its actual SearXNG
+  integration is the unrelated standalone `/v1/search` REST endpoint already
+  documented in `docs/research/litellm-searxng-search.md` (§1–3 there) — a
+  sibling API to chat/responses, not a Responses-API `tools:[{"type":"web_search"}]`
+  handler. It doesn't have a DashScope-mode either
+  (https://docs.litellm.ai/docs/providers/dashscope is a plain client wrapper
+  that calls the real dashscope.aliyuncs.com; nothing in it hosts a
+  DashScope-shaped server).
+- **LocalAI / Ollama**: no Responses API or DashScope-compatible mode found
+  in searches for either.
+- **A generic "OpenAI Responses API" self-hosted shim that could be relabeled**:
+  the closest match found, `teabranch/open-responses-server` (185 stars, 161
+  commits, wraps Ollama/vLLM as a Responses API with MCP support), **does not
+  implement `web_search` at all** — its own roadmap lists "Web search: crawl4ai"
+  as a *future* item, not shipped (verified live against the repo,
+  2026-09-05). No other candidate turned up in repeated GitHub searches for
+  "dashscope emulator/mock/fake server" or "responses api web_search
+  self-hosted."
+
+**Conclusion for §3: nothing installable off the shelf implements the
+DashScope Responses API's `web_search`/`web_extractor` hosted-tool surface.**
+Building it means writing your own small SSE server (see §5 sizing).
+
+## 4. Is DashScope's shape "OpenAI Responses API + web_search" reused wholesale?
+
+Yes, confirmed directly from source, not inference: qwen-code's client uses
+the **official `openai` npm package**'s `client.responses.create()` against a
+DashScope `baseURL` (§2 above) — it is not a DashScope-specific SDK or
+protocol. OpenAI's own Responses API supports a hosted `web_search_preview`
+tool with a similar `output[].type === "web_search_call"` item shape
+(OpenAI's public Responses API docs, referenced but not independently
+re-fetched here since qwen-code's source is authoritative for what it
+actually calls). DashScope's extension is the tool *name* (`web_search`
+rather than `web_search_preview` — the exact naming mismatch LiteLLM's own
+open bug in §3 stumbles on) plus the additional `web_extractor` tool and the
+`x_tools` usage-accounting field. No existing "OpenAI Responses API shim"
+project was found that already emulates `web_search_preview`/`web_search`
+server-side against a pluggable backend (see §3) — the two hosted-tool
+ecosystems (OpenAI's and DashScope's) both currently require literally
+calling out to the vendor's own cloud; nobody has open-sourced a
+self-hosted stand-in for either.
+
+## 5. LiteLLM specifically, re-examined against this exact requirement
+
+`docs/research/litellm-searxng-search.md` already established SearXNG is a
+first-class LiteLLM `search_provider` behind the **standalone** `/v1/search`
+REST endpoint (its own §1–2). That endpoint is irrelevant to qwen-code's
+`tools.webSearch.model` gate: qwen-code doesn't call an arbitrary search REST
+endpoint, it calls `POST {baseUrl}/responses` on an **OpenAI-SDK client**
+with `tools:[{type:"web_search"}]`, and gates `baseUrl` on the Alibaba
+hostname allowlist in §2. Even ignoring the hostname gate entirely (i.e.
+assuming a patched qwen-code build), LiteLLM's `/v1/responses` route
+currently **rejects** the `web_search` tool type per the open bug in §3 — so
+today, LiteLLM cannot terminate this request even as an internal component of
+a custom build. Nothing here changes the litellm-searxng-search.md
+recommendation; it remains correct and unrelated to this question.
+
+## 6. Effort assessment and recommendation
+
+**Option A — patch qwen-code + hand-roll a DashScope-Responses-shaped SSE
+server in front of SearXNG.** What it needs, concretely:
+1. Fork qwen-code, delete/relax `classifyDashScopeBaseUrl()` (§2) — trivial,
+   but means building and distributing a patched CLI, and re-patching on every
+   upstream update that touches this file or its surrounding gate logic.
+2. Write a small HTTP server exposing `POST /responses` that: accepts the
+   exact request shape in §2, calls SearXNG (`http://search.home`, already
+   reachable per `docs/research/litellm-searxng-search.md`'s `extra_hosts`
+   finding) for results, and streams back SSE events in the precise sequence
+   qwen-code's parser expects (`response.output_item.added` /
+   `.done` with a `web_search_call` item carrying `action.sources[].url`,
+   optionally a `message` item with narrated text, then
+   `response.completed`). No narration/LLM step is strictly required — an
+   empty or templated `message` still satisfies the parser as long as at
+   least one non-`failed` `web_search_call` item exists (§2's "no-search"
+   check only counts search-call items, not narration quality).
+   Realistically a few hundred lines (Node/Python + SSE), a day or so of
+   work plus debugging the exact event ordering, error-shape (`event:error`
+   quirk), and `store`/`instructions` fields the client sends but doesn't
+   strictly require echoing back.
+3. Register this server's URL as a `modelProviders` entry — except the
+   patched hostname check from step 1 is required for step 3 to pass at all,
+   so steps 1 and 2 are both mandatory, not alternatives.
+4. Maintain the fork indefinitely against upstream qwen-code releases.
+
+**Option B — do nothing further.** `docs/research/omniroute-qwen-websearch.md`
+already documents a **verified, working, fully self-hosted** path: OmniRoute's
+own `omniroute_web_search` MCP tool, backed by this stack's SearXNG instance,
+confirmed connected (`qwen mcp list` → Connected) and exercised end-to-end
+(`POST /v1/search` returned real results). This uses qwen-code's *documented,
+supported, unpatched* MCP-server extension point (`mcpServers` in
+`settings.json`) — no fork, no upstream-drift risk, no protocol shape to
+maintain.
+
+**Recommendation: do not build Option A.** The built-in `web_search` tool's
+"DashScope-compatible" requirement is, by design in qwen-code's own source, a
+hostname allowlist for Alibaba's cloud — it is not a compatibility surface
+meant to be reimplemented, and no one else has reimplemented it either (§3).
+Satisfying it self-hosted requires forking and permanently maintaining a
+patch to code whose only purpose is to *stop* you from doing that. The MCP
+path in `omniroute-qwen-websearch.md` already delivers the same end-user
+capability (web search, backed by this stack's own SearXNG, no external
+API) through qwen-code's actual supported extension point, with zero ongoing
+fork-maintenance burden. There is no functional gap Option A would close that
+Option B doesn't already close today.
+
+## Open questions / unknowns
+
+- Whether `DASHSCOPE_REGIONAL_HOSTS` or the extra suffixes
+  (`maas.aliyuncs.com`, `alibaba-inc.com`, `aliyun-inc.com`) ever change
+  across qwen-code releases — checked only against the current `main` branch
+  (fetched 2026-09-05); a future release could tighten or loosen this list.
+- Whether OpenAI's own `web_search_preview` Responses-API tool has a
+  publicly documented exact request/response JSON schema identical enough to
+  DashScope's `web_search`/`web_extractor` pair that a single shim could serve
+  both — not independently verified against OpenAI's own docs in this pass;
+  qwen-code's source (§2) is authoritative for the DashScope side only.
+- Whether `teabranch/open-responses-server`'s planned "Web search: crawl4ai"
+  roadmap item, if shipped, would end up emitting DashScope-shaped
+  `web_search_call` items or OpenAI-shaped `web_search_preview` ones — could
+  become relevant later but is speculative (unshipped) as of this research.
+
+## Sources
+
+- https://qwenlm.github.io/qwen-code-docs/en/developers/tools/web-search/ and
+  https://raw.githubusercontent.com/QwenLM/qwen-code/main/docs/developers/tools/web-search.md
+  — current built-in-tool vs. MCP options, settings keys, migration note.
+- `packages/core/src/tools/web-search.ts`,
+  `packages/core/src/core/openaiContentGenerator/constants.ts`,
+  `packages/core/src/core/openaiContentGenerator/provider/dashscope.ts` —
+  fetched directly from `QwenLM/qwen-code`'s `main` branch via
+  `raw.githubusercontent.com` on 2026-09-05; ground truth for the request
+  shape, SSE parsing, and the hostname gate (§2).
+- https://github.com/QwenLM/qwen-code/issues/3841 — prior (closed,
+  "not planned") community proposal for DashScope `enable_search` passthrough;
+  shows the feature that eventually shipped took a different path (Responses
+  API, not Chat Completions `enable_search`).
+- https://www.alibabacloud.com/help/en/model-studio/qwen-api-via-openai-responses
+  and https://www.alibabacloud.com/help/en/model-studio/compatibility-with-openai-responses-api
+  — Alibaba's own Responses API docs: endpoint, `tools` shape, `output` item
+  shape.
+- https://docs.qwencloud.com/developer-guides/tool-calling/web-search —
+  the older Chat-Completions `enable_search` mechanism and its
+  no-citations limitation.
+- https://docs.vllm.ai/en/stable/api/vllm/entrypoints/openai/responses/,
+  https://vllm.ai/blog/2025-08-05-gpt-oss,
+  https://github.com/vllm-project/recipes/blob/main/OpenAI/GPT-OSS.md — vLLM's
+  Responses API and gpt-oss browser-tool/tool-server support.
+- https://github.com/sgl-project/sglang/issues/10038 — SGLang Responses API
+  support unclear.
+- https://github.com/BerriAI/litellm/issues/14011 — LiteLLM's `/v1/responses`
+  rejects the `web_search` tool type.
+- https://docs.litellm.ai/docs/providers/dashscope — LiteLLM's DashScope
+  provider is a plain client wrapper, no Responses API, no web_search.
+- https://github.com/teabranch/open-responses-server — closest
+  "self-hosted Responses API shim" found; web_search not implemented
+  (roadmap item only), checked live 2026-09-05.
+- `G:\_DEV\repos\LLM-Server\docs\research\omniroute-qwen-websearch.md` —
+  the already-working, verified self-hosted alternative this doc is weighed
+  against.
+- `G:\_DEV\repos\LLM-Server\docs\research\litellm-searxng-search.md` —
+  LiteLLM's actual (unrelated) SearXNG integration, re-confirmed as
+  orthogonal to this question in §5.
+
+## Tried it live (2026-09-05) — confirmed empirically, plus one new fact
+
+The user asked to actually run the experiment rather than stop at the analysis above.
+
+**What was done** (all local to the WSL install, reverted afterward — nothing in
+this repo or the live OmniRoute instance was left changed):
+- Patched the installed CLI file
+  `~/.local/lib/qwen-code/lib/chunks/web-search-K2FMOGS5.js` with a one-line
+  bypass in `classifyDashScopeBaseUrl()`: `if (baseUrl.includes("proxy-ai.home")) return null;`
+- Added a `tools.webSearch` block to `~/.qwen/settings.json` pointing
+  `model`/`baseUrl` at a new `qwen-experiment-websearch` `modelProviders` entry
+  using OmniRoute's existing `http://proxy-ai.home/v1` and the already-working
+  `OMNIROUTE_API_KEY`.
+- Ran `qwen` with a prompt forcing use of the built-in `web_search` tool.
+
+**Result — the client-side gate bypass worked**, confirming the research's
+read of `classifyDashScopeBaseUrl()` was accurate: qwen accepted the OmniRoute
+host as "DashScope-compatible" and attempted the tool call. It stopped at an
+interactive approval prompt first (expected — headless auto-approve wasn't
+attempted, since that flips on unrestricted auto-execution of every tool call
+at process privilege, not just this one).
+
+**New fact, not visible from static docs alone**: a direct `curl -X POST
+http://proxy-ai.home/v1/responses` (with a valid key, matching the request
+shape qwen would send) returned `{"error":{"message":"No active credentials
+for provider: codex.","type":"authentication_error","code":"invalid_api_key"}}`
+— **not** the generic "unknown route" error a nonexistent path returns (verified
+earlier in this same research thread against `/v1/search`-adjacent bogus
+paths). So `/v1/responses` **is a real, implemented OmniRoute route**, not
+merely undocumented — the earlier inference that it didn't exist was wrong;
+it exists but is hardcoded to proxy exclusively through a specific provider
+connection OmniRoute's catalog calls `codex`.
+
+**`codex` identified via `PROVIDER_REFERENCE.md`**: `id: codex`, alias `cx`,
+name "OpenAI Codex", **auth type: OAuth** — a real, personal
+ChatGPT/OpenAI-account connection, not a free/no-auth scraper provider like
+several others already connected in this instance (`felo-web`,
+`duckduckgo-web`, etc.). Checked `docs/reference/ENVIRONMENT.md` for any
+setting to redirect `/v1/responses` to a different provider — **none
+exists**; there is no `responsesProvider` or equivalent override.
+
+**Why routing isn't configurable, architecturally**: OpenAI's Responses API
+`web_search` is a *hosted* tool — the search executes inside the model
+backend's own infrastructure as part of generating the response, not as a
+client-visible round trip. Confirmed directly against llama.cpp's own
+`tools/server` docs (`github.com/ggml-org/llama.cpp/tree/master/tools/server`):
+it implements only `/v1/chat/completions` with client-side tool-calling
+(the model emits a `tool_call`; the *client* must execute it), has no
+`/v1/responses` endpoint, no hosted-tool execution, and its built-in
+`--tools` are local-only (`read_file`, `grep_search`, `exec_shell_command`,
+etc.) — none make outbound HTTP requests. So even with configurable routing,
+pointing `/v1/responses` at the local Qwen model wouldn't work: the upstream
+llama-server has nothing that could serve the hosted-tool half of the
+contract. Building that would mean OmniRoute (or a custom shim) intercepting
+the model's tool-call mid-generation and splicing in a real search — the
+same shim work priced out as not-worth-it earlier in this document, now
+confirmed to be the *only* way, not one option among several.
+
+**Conclusion holds, sharpened**: the dead end isn't just qwen-code's
+client-side hostname check anymore — even a fully self-hosted, hostname-gate-bypassed
+setup terminates at OmniRoute's `codex`-only `/v1/responses` routing, which
+itself terminates at needing a real OpenAI/ChatGPT OAuth account, which is
+exactly the kind of external paid dependency this whole line of inquiry was
+trying to avoid. `omniroute_web_search` via MCP (already working, already
+free, already self-hosted) remains the only path that actually satisfies the
+original goal.
+
+**Revert**: both the CLI patch and the `settings.json` changes were reverted
+after the test — `omniroute-search` MCP confirmed still `Connected` via
+`qwen mcp list` afterward. No lasting changes from this experiment.
@@ -0,0 +1,221 @@
+# Which small model to run as the always-resident `fastModel` for qwen-code's Auto Mode classifier?
+
+**Date:** 2026-09-06
+**Budget:** ≤7GB VRAM, resident concurrently alongside the existing Qwen3.8-27B instance on the single
+32GB R9700, via the same `llama.cpp:server-rocm` image already in `docker-compose.yml`.
+**Answer: Qwen3-4B-Instruct-2507, Q8_0 GGUF (~4.3GB weights).** The prior quick pass's tentative pick
+holds up under primary-source verification, for a more specific reason than "same tokenizer family":
+it is the only strong candidate in the shortlist that is *architecturally* non-thinking (no `<think>`
+code path exists at all, vs. models that are thinking-by-default and rely on a per-call
+`enable_thinking:false` toggle that llama.cpp does not cleanly expose). It does carry one directly
+relevant, documented llama.cpp bug — but that bug is closed, has a one-flag workaround, and is
+strictly less severe than the still-open Qwen3.5/Qwen3.8-lineage bugs already documented against the
+27B model in this repo.
+
+## 1. What the classifier actually needs (grounding the requirement)
+
+Per qwen-code's own docs, Auto Mode's permission gate is a two-stage LLM classifier:
+
+- **Stage 1** — outputs only `{ shouldBlock: bool }`, ~300ms budget, thinking already disabled at the
+  request level. If `shouldBlock` is `false`, the action proceeds immediately.
+- **Stage 2** — only runs when Stage 1 blocks; uses chain-of-thought review to downgrade false
+  positives, ~3-5s budget.
+- Both stages use "your configured fast model (`/model --fast`)"; if none is configured, the full
+  session model is used instead — which is the current, too-slow state this second model is meant to
+  fix.
+
+Source: [Qwen Code docs — Auto Mode](https://qwenlm.github.io/qwen-code-docs/en/users/features/auto-mode/),
+[QwenLM/qwen-code docs/users/features/auto-mode.md](https://github.com/QwenLM/qwen-code/blob/main/docs/users/features/auto-mode.md).
+
+A live qwen-code issue independently confirms the exact failure mode this repo already hit with the
+27B model — a model *thinking* inside the classifier path is a first-order latency problem, not a
+nice-to-have to tune later:
+
+> "for a latency-sensitive permission gate, thinking should be disabled in every stage" — enabling it
+> "makes the review path slower and more expensive, which directly worsens the timeout problem."
+
+That issue (timeouts tripping on slow inference) was closed by a PR that both loosened the stage
+timeout budgets *and* moved toward disabling thinking everywhere in the classifier.
+Source: [QwenLM/qwen-code issue #4676](https://github.com/QwenLM/qwen-code/issues/4676).
+
+Takeaway for model selection: the request-level "don't think" instruction already exists in
+qwen-code's own classifier code. What matters is whether the **model + llama.cpp combination actually
+honors it reliably** — which is precisely where the 27B model failed (see
+[`qwen3.8-27b-tool-calling.md`](qwen3.8-27b-tool-calling.md)) and where several shortlist candidates
+have their own version of the same problem.
+
+## 2. Candidates evaluated against primary sources
+
+| Model | Params | GGUF size (quant) | Context | License | Thinking behavior | Tool-calling | Verdict |
+|---|---|---|---|---|---|---|---|
+| **Qwen3-4B-Instruct-2507** | 4B | Q4_K_M 2.5GB / **Q8_0 4.28GB** | 262,144 native | Apache 2.0 | **Non-thinking only** — model card states it "does not generate `<think></think>` blocks in its output," full stop, no toggle needed | Yes, native `<tool_call>` format, BFCL-v3 61.9 | **Recommended** |
+| Qwen3-1.7B | 1.7B | ~1.1GB (Q4_K_M, typical) | 32,768 | Apache 2.0 | Thinking **on by default**; needs `enable_thinking:false` per call | Yes | Rejected — see §3 |
+| Qwen3-0.6B | 0.6B | ~0.4GB (Q4_K_M) | 32,768 | Apache 2.0 | Thinking on by default, same toggle issue as 1.7B | Yes, but weakest reasoning of the family | Rejected — undersized for reliability at this size, same toggle risk |
+| Llama-3.2-3B-Instruct | 3B | ~2GB (Q4_K_M, typical) | 128K | Llama 3.2 Community License — commercial use allowed, but text/EU carve-out language and an explicit >700M-MAU re-licensing clause | No thinking mode | Not natively documented on the model card fetched (no tool-call format called out) | Deprioritized — license has more fine print than Apache 2.0 for no clear benefit here |
+| Gemma-3-4b-it | 4B | Q4_K_M 2.49GB / Q8_0 4.13GB | 128K | Custom "Gemma" license (Google usage terms) | No documented thinking mode | Not documented on the model card fetched | Deprioritized — no confirmed native tool-calling story, non-Apache license |
+| Phi-4-mini-instruct | 3.8B | Q4_K_M 2.49GB / Q8_0 4.08GB | 128K | **MIT** | Not a reasoning model (that's the separate Phi-4-mini-**reasoning** model); no `<think>` tags by default | Yes — documented function-call format with dedicated tokens | Credible alternative — see §4 |
+| SmolLM3-3B | 3B | Q4_K_M ~1.9GB (typical) | 128K (64K trained + YaRN) | Apache 2.0 | **Thinking on by default** (`enable_thinking`), toggled via system-prompt flags | Yes (XML or Python-style tool calls) | Rejected — same thinking-by-default risk as Qwen3-1.7B |
+| Ministral-8B-Instruct-2410 | 8B | too large for budget at any useful quant with headroom | 128K | **Mistral Research License — commercial use requires contacting Mistral for a separate license** | Not documented as a reasoning model | Yes, documented function-calling with benchmark (31.6 vs Mistral-7B's 6.9) | Rejected — license restricts this repo's own dev-tooling use without contacting Mistral; also parameter count crowds the 7GB budget once Q8_0 + KV cache is counted |
+
+Sources (fetched directly from each model's own HF card / GGUF repo unless noted):
+[Qwen/Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507),
+[unsloth/Qwen3-4B-Instruct-2507-GGUF](https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF),
+[Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B),
+[Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B),
+[meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct),
+[google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it),
+[bartowski/google_gemma-3-4b-it-GGUF](https://huggingface.co/bartowski/google_gemma-3-4b-it-GGUF),
+[microsoft/Phi-4-mini-instruct](https://huggingface.co/microsoft/Phi-4-mini-instruct),
+[bartowski/microsoft_Phi-4-mini-instruct-GGUF](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF),
+[HuggingFaceTB/SmolLM3-3B](https://huggingface.co/HuggingFaceTB/SmolLM3-3B),
+[mistralai/Ministral-8B-Instruct-2410](https://huggingface.co/mistralai/Ministral-8B-Instruct-2410).
+
+**Confidence note:** file sizes for Qwen3-1.7B/0.6B, Llama-3.2-3B, and SmolLM3-3B GGUF quants above are
+typical/approximate — those repos weren't individually re-verified against a specific GGUF file tree
+since all three families were eliminated on architectural grounds (§3) before size mattered. Sizes for
+the two models actually compared head-to-head (Qwen3-4B-Instruct-2507, Phi-4-mini-instruct) and
+Gemma-3-4b-it were pulled directly from each quantizer's own repo page.
+
+## 3. Why "thinking-by-default + per-call toggle" is disqualifying, not just a minor ding
+
+This is the deciding architectural distinction, and it's exactly the failure this second model exists
+to avoid. Qwen's own llama.cpp docs page states the toggle problem directly:
+
+> "the hard switch implemented in the chat template is not exposed in llama.cpp" for controlling
+> `enable_thinking` — the documented workaround is to supply "a custom chat template equivalent to
+> always `enable_thinking=False`" via `--chat-template-file`.
+Source: [Qwen — Run with llama.cpp](https://qwen.readthedocs.io/en/latest/run_locally/llama.cpp.html).
+
+That means for Qwen3-1.7B, Qwen3-0.6B, and SmolLM3-3B — all thinking-on-by-default — reliably
+suppressing the reasoning phase in this llama.cpp/ROCm stack is not a request-body flag away; it needs
+a hand-maintained custom chat template file, which is exactly the kind of fragile, easy-to-silently-
+regress setup this task is trying to get away from (the 27B model's whole problem was reasoning_content
+being consumed before the answer). Qwen3-4B-Instruct-2507 has no such toggle to maintain in the first
+place — the model card states the non-thinking behavior as an unconditional property of the model, not
+a configurable default that has to be forced correctly on every request. This is a stronger claim than
+"same tokenizer family as the 27B" (the original quick-pass's reasoning) and is the actual basis for
+the recommendation.
+
+## 4. The one documented risk specific to Qwen3-4B-Instruct-2507 — and why it doesn't change the pick
+
+llama.cpp has its own closed, dated bug where server builds around **b8429** (March 2026)
+mis-detected Qwen3-Instruct-2507 models — the 4B included by name in the reporter's repro command — as
+thinking models, routing tool-call output into `reasoning_content` instead of `tool_calls`:
+
+> "llama.cpp b8429 incorrectly detects Qwen3-Instruct-2507 models as thinking models (`thinking = 1`).
+> This causes tool calls to be captured as `reasoning_content` instead of being parsed into the
+> `tool_calls` array."
+
+The documented, confirmed-working workaround is a single server flag:
+
+```
+llama-server -hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M --jinja --port 8222 --reasoning off
+```
+
+which restores `thinking = 0` and correct `finish_reason: tool_calls` output. The issue is **closed**.
+Source: [ggml-org/llama.cpp issue #20809](https://github.com/ggml-org/llama.cpp/issues/20809).
+
+This is worth flagging honestly against the recommendation, but it's materially different from the
+open, only-partially-fixed Qwen3.5/Qwen3.8-lineage parser bugs already documented in this repo's
+[`qwen3.8-27b-tool-calling.md`](qwen3.8-27b-tool-calling.md) (issues #21158, #20837 — both open at time
+of that research): this is a llama.cpp *server-side misdetection* bug with a one-flag fix, not an
+unresolved upstream grammar/parser defect in the Qwen3.5 architecture family itself. Concretely: add
+`--reasoning off` to this second llama-server instance's command regardless — it's a no-cost safety net
+whether or not the current `ghcr.io/ggml-org/llama.cpp:server-rocm` build still has the bug, and it
+directly targets the exact failure mode (reasoning_content eating the completion) that ruled out the
+27B model for this role in the first place.
+
+## 5. VRAM math for the classifier role specifically
+
+Qwen3-4B-Instruct-2507 is a plain (non-hybrid) transformer — every layer is standard GQA attention, so
+unlike the 27B model's Gated-DeltaNet hybrid, KV cache scales with *all* layers, not a fraction of them.
+From the model's own `config.json`:
+
+- `num_hidden_layers`: 36, `num_key_value_heads`: 8, `head_dim`: 128
+Source: [Qwen/Qwen3-4B-Instruct-2507 config.json](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/raw/main/config.json).
+
+Per-token KV cache (fp16, both K and V):
+`36 layers × 2 (K+V) × 8 kv_heads × 128 head_dim × 2 bytes = 144 KiB/token`
+
+The classifier transcript is bounded by design — qwen-code's own two-stage design keeps Stage 1 to a
+`{shouldBlock}`-only judgment and Stage 2 to a chain-of-thought review of one blocked action, not an
+open-ended agent session — so a context window in the low thousands of tokens is generous headroom,
+not a tight fit:
+
+| Context | KV cache (fp16) | KV cache (q8_0, `--cache-type-k/v q8_0`) | Weights (Q8_0) | Total (q8_0 KV) | Headroom under 7GB |
+|---|---|---|---|---|---|
+| 4,096 tokens | ~0.56 GB | ~0.28 GB | 4.28 GB | **~4.56 GB** | ~2.4 GB |
+| 8,192 tokens | ~1.13 GB | ~0.56 GB | 4.28 GB | **~4.84 GB** | ~2.2 GB |
+| 32,768 tokens (generous ceiling) | ~4.5 GB | ~2.25 GB | 4.28 GB | **~6.53 GB** | ~0.5 GB (tight) |
+
+At any context length actually needed for a permission-gate classifier (thousands, not tens of
+thousands, of tokens), Q8_0 weights plus q8_0 KV cache comfortably clears the 7GB ceiling with headroom
+to spare for the compute buffer and batch overhead — matching the same `--cache-type-k q8_0
+--cache-type-v q8_0` pattern this repo already uses for the 27B instance. There's no need to drop to
+Q4_K_M (2.5GB) unless a much larger classifier context is anticipated later; Q8_0 is the better default
+here since it's a small model where quantization loss matters proportionally more, and the VRAM budget
+comfortably affords the higher-precision quant.
+
+## 6. What would change the answer
+
+- **If Phi-4-mini-instruct's MIT license matters more than matching the 27B model's tokenizer/template
+  family**, it's a legitimate second choice: confirmed non-thinking by default, confirmed native
+  function-calling format, comparable Q8_0 size (4.08GB), and a license with zero commercial-use fine
+  print (vs. Apache 2.0's still-permissive but slightly more conditional terms). It wasn't picked
+  because it has no llama.cpp-specific tool-calling track record verified in this pass (no equivalent
+  to the issue #20809 workaround search done for it), so its actual reliability on this exact
+  `llama.cpp:server-rocm` stack is less directly evidenced than Qwen3-4B-Instruct-2507's.
+- **If the classifier transcript ever needs to grow well past ~8K tokens routinely**, drop to Q4_K_M
+  (2.5GB) to keep well clear of the 7GB ceiling — the KV-cache math in §5 shows the crossover point.
+- **If llama.cpp's #20809 misdetection turns out to still reproduce** on the exact
+  `ghcr.io/ggml-org/llama.cpp:server-rocm` build this repo pulls, the fix is the one-flag
+  `--reasoning off` workaround already confirmed in that issue — not a reason to pick a different
+  model, since every thinking-capable alternative in this shortlist has an equal-or-worse version of
+  the same class of bug with less clean workarounds (custom chat-template files, per §3).
+
+## Sources
+
+- [Qwen Code docs — Auto Mode](https://qwenlm.github.io/qwen-code-docs/en/users/features/auto-mode/)
+- [QwenLM/qwen-code — docs/users/features/auto-mode.md](https://github.com/QwenLM/qwen-code/blob/main/docs/users/features/auto-mode.md)
+- [QwenLM/qwen-code issue #4676](https://github.com/QwenLM/qwen-code/issues/4676)
+- [Qwen/Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507)
+- [Qwen/Qwen3-4B-Instruct-2507 config.json](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/raw/main/config.json)
+- [unsloth/Qwen3-4B-Instruct-2507-GGUF](https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF)
+- [Qwen — Run with llama.cpp](https://qwen.readthedocs.io/en/latest/run_locally/llama.cpp.html)
+- [ggml-org/llama.cpp issue #20809](https://github.com/ggml-org/llama.cpp/issues/20809)
+- [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B)
+- [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B)
+- [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct)
+- [google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it)
+- [bartowski/google_gemma-3-4b-it-GGUF](https://huggingface.co/bartowski/google_gemma-3-4b-it-GGUF)
+- [microsoft/Phi-4-mini-instruct](https://huggingface.co/microsoft/Phi-4-mini-instruct)
+- [bartowski/microsoft_Phi-4-mini-instruct-GGUF](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF)
+- [HuggingFaceTB/SmolLM3-3B](https://huggingface.co/HuggingFaceTB/SmolLM3-3B)
+- [mistralai/Ministral-8B-Instruct-2410](https://huggingface.co/mistralai/Ministral-8B-Instruct-2410)
+- [docs/research/qwen3.8-27b-tool-calling.md](qwen3.8-27b-tool-calling.md) (this repo — cross-referenced
+  for the 27B model's own, still-open, tool-calling parser bugs)
+
+## Confidence/uncertainty summary
+
+- **High confidence:** Qwen3-4B-Instruct-2507's non-thinking-only status (direct model-card quote);
+  qwen-code's two-stage classifier timing/design and its use of `/model --fast` (direct docs quote);
+  the existence, exact symptom, and workaround of llama.cpp issue #20809 (direct issue quote); the
+  KV-cache architecture math (computed directly from the model's own `config.json`, same method as the
+  existing `qwen3.8-27b-quant.md` research in this repo).
+- **Medium confidence:** exact GGUF file sizes for Qwen3-1.7B, Qwen3-0.6B, Llama-3.2-3B-Instruct, and
+  SmolLM3-3B — not individually re-verified against a specific quantizer's file tree since these were
+  eliminated on architectural (thinking-toggle) grounds before size became the deciding factor; treat
+  as typical/approximate, not exact.
+- **Low confidence / not independently verified:** whether the current
+  `ghcr.io/ggml-org/llama.cpp:server-rocm` image (pulled fresh) still reproduces issue #20809's
+  misdetection — the issue is closed but no changelog/PR diff was fetched to confirm the underlying
+  detection logic was actually patched vs. the reporter simply adopting the `--reasoning off`
+  workaround. Recommend a live smoke test (send one tool-calling request, confirm the response lands in
+  `tool_calls` not `reasoning_content`, and time a trivial completion) before wiring this model in as
+  the production `fastModel`, the same caveat this repo's `qwen3.8-27b-tool-calling.md` already flags
+  for the 27B model.
+- Whether Phi-4-mini-instruct has a comparably clean llama.cpp tool-calling track record was **not**
+  deep-dived (no issue-tracker search run against it) — it's flagged in §6 as a live alternative rather
+  than fully evaluated, since Qwen3-4B-Instruct-2507's architectural non-thinking guarantee and
+  same-family template consistency with the existing 27B deployment made it the clearer pick without
+  needing that extra research pass.
@@ -0,0 +1,220 @@
+# Research: which diffusion model to target with the full ~32GB R9700
+
+**Question:** With Qwen/llama-server fully stopped (per issue #39's premise — see
+map #38), ComfyUI has the whole ~32GB R9700 (ROCm/gfx1201) to itself instead
+of the ~6GB left over during concurrent operation
+(per [`image-generation-options.md`](image-generation-options.md)). Given that
+headroom, should the build stay on FLUX.1-schnell, or move up to
+FLUX.1-dev, SD3.5-large, Qwen-Image, HunyuanImage-3.0, or Krea-2?
+
+**Answer, short version:** move up to **Qwen-Image at FP8 precision**
+(`qwen_image_fp8_e4m3fn.safetensors` diffusion weights +
+`qwen_2.5_vl_7b_fp8_scaled.safetensors` text encoder, ~25GB combined). It's
+the only one of the five upgrade candidates with **direct, hardware-specific
+evidence of running on this exact GPU architecture** (gfx1201/R9700) rather
+than a generic "ComfyUI supports ROCm" inference, it carries the cleanest
+license of the group (Apache-2.0, no revenue threshold, no non-commercial
+clause), and its 20B MMDiT is a real capability step up from schnell's
+distilled 12B (notably for text rendering and prompt adherence), while still
+fitting with real margin inside 32GB.
+
+## Why not just re-derive the schnell/dev/SDXL/SD3.5 findings
+
+`docs/research/image-generation-options.md` already covers, with primary
+sources: ComfyUI's ROCm/gfx1201 story (official AMD docs + RDNA4 blog post +
+community gfx1201 Docker images), FLUX.1-schnell vs FLUX.1-dev vs SDXL vs
+SD3.5 licenses, and FLUX GGUF VRAM figures at the ~6GB-headroom scale. None
+of that is repeated here except where the ~32GB ceiling changes the
+conclusion. This doc adds: FLUX.1-dev/SD3.5 at the *larger* headroom, plus
+three models the prior doc didn't cover at all (Qwen-Image, HunyuanImage-3.0,
+Krea-2).
+
+## Candidate comparison
+
+| Model | License (primary source) | Params | Stated/typical VRAM | ROCm/gfx1201 evidence |
+|---|---|---|---|---|
+| FLUX.1-schnell (current) | Apache-2.0 | 12B | GGUF Q4_K_S ~7GB (per prior doc) | Confirmed on gfx1201 (prior doc) |
+| **Qwen-Image** | **Apache-2.0** | 20B (20.4B DiT + 8.3B Qwen2.5-VL text encoder) | fp8 ~16GB (diffusion) + ~9.4GB (fp8 text encoder) ≈ 25GB total; bf16 needs 24GB+ and "48GB+" per some quant write-ups | **Direct**: [kyuz0/amd-r9700-comfy](https://github.com/kyuz0/amd-r9700-comfy) ships a pre-validated "Qwen Image 2512 (FP8) & Lightning LoRA (4 steps)" ComfyUI workflow specifically for the R9700 AI Pro (gfx1201), on a ROCm 7 (TheRock nightlies) toolbox |
+| FLUX.1-dev | [FLUX.1-dev Non-Commercial License](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md) — non-commercial weights, outputs usable commercially | 12B | bf16 ~24GB; GGUF Q8 ~12-13GB | Same *family* evidence as schnell (gfx1201 Docker images target FLUX generally), but no R9700-specific FLUX.1-dev report found |
+| SD3.5-large | [Stability Community License](https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/LICENSE.md) — free under $1M annual revenue | 8B | bf16 ~16GB; 4-bit NF4 fits small GPUs | AMD's own ComfyUI-ROCm doc lists an "SD3.5 Simple" template workflow (per prior doc) — vendor-blessed but not R9700-specific |
+| HunyuanImage-3.0 | [tencent-hunyuan-community license](https://github.com/Tencent-Hunyuan/HunyuanImage-3.0) | 80B total / 13B active (MoE, 64 experts) | Official repo: "≥ 3 × 80GB" VRAM for the base model, "≥ 8 × 80GB" for -Instruct; ~177GB at fp16 | **None, and actively contraindicated**: setup requires CUDA 12.8 + FlashAttention2/FlashInfer, no AMD/ROCm mention anywhere in the official repo |
+| Krea-2 (Turbo) | [Krea 2 Community License](https://www.krea.ai/krea-2-licensing) — free under $1M annual revenue, homelab/personal explicitly covered | 12-13B DiT | No official VRAM figure; community reports (RTX hardware only) cite fp8 ~16GB, GGUF ~12GB | **None found** — released [June 22, 2026 per the HF model card](https://huggingface.co/krea/Krea-2-Turbo); ComfyUI added native support per [blog.comfy.org](https://blog.comfy.org/p/krea-2-open-source-models-are-now), and GGUF quants exist ([molbal/krea2-gguf](https://huggingface.co/molbal/krea2-gguf)), but no R9700/gfx1201-specific report exists yet — too new for that evidence to have accumulated |
+
+## Per-model detail
+
+### Qwen-Image — recommended
+
+- **License**: Apache-2.0, stated directly on the model card, no revenue
+  threshold, no non-commercial clause, no attribution/naming requirement.
+  The cleanest license of every model considered in this doc or the prior
+  one. Source: [Qwen/Qwen-Image on Hugging Face](https://huggingface.co/Qwen/Qwen-Image).
+- **Architecture**: 20B-parameter MMDiT (Multimodal Diffusion Transformer)
+  combined with an 8.3B Qwen2.5-VL text encoder — notably larger and more
+  capable than FLUX.1-schnell's 12B distilled model, particularly for
+  multilingual text rendering and instruction-following, per the official
+  [QwenLM/Qwen-Image GitHub repo](https://github.com/QwenLM/Qwen-Image).
+- **ComfyUI support**: native, not a wrapper/custom-node integration —
+  landed August 2025 per [ComfyUI Wiki's native-support announcement](https://comfyui-wiki.com/en/news/2025-08-05-qwen-image),
+  with an official FP8 checkpoint (`qwen_image_fp8_e4m3fn.safetensors`) and
+  FP8-scaled text encoder (`qwen_2.5_vl_7b_fp8_scaled.safetensors`, 9.38GB)
+  published under [Comfy-Org/Qwen-Image_ComfyUI](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders).
+- **VRAM at FP8**: the official Comfy-Org FP8 diffusion checkpoint plus FP8
+  text encoder land around ~25GB combined — comfortably inside the 32GB
+  ceiling with headroom for ComfyUI runtime/VAE overhead, versus 24GB+ (some
+  sources say 48GB+) for the unquantized bf16 model. Sources: community
+  VRAM write-ups aggregated via [Comfy-Org/ComfyUI issue #10852 ("Qwen-image in 24GB VRAM and 32GB RAM")](https://github.com/Comfy-Org/ComfyUI/issues/10852)
+  and the [official ComfyUI Qwen-Image example](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/) —
+  treat exact GB figures as secondary/community-sourced, consistent with how
+  the prior doc flagged FLUX's GGUF VRAM table.
+- **Direct R9700/gfx1201 evidence (the deciding factor)**:
+  [kyuz0/amd-r9700-comfy](https://github.com/kyuz0/amd-r9700-comfy) is a
+  Fedora-toolbox ROCm 7 (TheRock nightlies) environment built specifically
+  for the "AMD Radeon 9700 AI PRO (32GB)" and ships a pre-validated
+  "Qwen Image 2512 (FP8) & Lightning LoRA (4 steps)" ComfyUI workflow
+  (plus a Qwen-Image-Edit 2511 FP8 workflow) as one of only four workflows
+  in the whole repo. This is the only model in this comparison with
+  card-architecture-specific (not just "ComfyUI supports ROCm generically")
+  validation — everything else relies on family-level or vendor-generic
+  ROCm claims. A related community discussion
+  ([pollockjj/ComfyUI-MultiGPU #133](https://github.com/pollockjj/ComfyUI-MultiGPU/discussions/133))
+  does flag that "Qwen image edit doesn't always work on AMD HIP/ROCm" in
+  some configurations — worth testing the specific workflow before assuming
+  zero friction, but this is a known-quantity, actively-discussed rough edge
+  rather than a documented hard blocker.
+- **4-step Lightning LoRA**: the R9700-validated workflow pairs Qwen-Image
+  with a "Lightning" LoRA for 4-step inference — the same fast-inference
+  pattern FLUX.1-schnell uses, so switching models doesn't have to mean
+  giving up the short GPU-resident-time-per-image property the prior doc
+  called out as valuable for time-sliced use alongside llama-server (even
+  though this ticket's premise is llama-server being stopped, that pattern
+  still helps if the two services are ever run in an overlapping window).
+
+### FLUX.1-dev — solid alternative, not the pick
+
+- License permits non-commercial use of the weights; BFL's own license page
+  states generated outputs are separately usable commercially (already
+  covered in the prior doc; unchanged here). Source:
+  [FLUX.1-dev LICENSE.md](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md).
+- At bf16 (~24GB) or GGUF Q8 (~12-13GB), it fits the 32GB ceiling with room
+  to spare — a legitimate move up from schnell's distilled quality.
+  Source: [city96/FLUX.1-dev-gguf](https://huggingface.co/city96/FLUX.1-dev-gguf)
+  and community Q8 VRAM reports.
+- Loses to Qwen-Image on two counts: license (non-commercial-weights clause
+  vs. Apache-2.0 — not a hard blocker for this homelab per the prior doc's
+  own reasoning, but strictly worse) and hardware evidence (FLUX's gfx1201
+  validation is at the *family* level — the prior doc's `yurisasc/comfyui-rocm-rdna4`
+  and `charlie12345/R9700AIProComfyUIPatch` sources are about running FLUX
+  models on this card generally, not a FLUX.1-dev-specific report the way
+  kyuz0's repo is Qwen-Image-specific).
+
+### SD3.5-large — no longer the best use of the freed headroom
+
+- License unchanged from the prior doc: Stability Community License, free
+  under $1M annual revenue (irrelevant threshold for this homelab). Source:
+  [stabilityai/stable-diffusion-3.5-large LICENSE.md](https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/LICENSE.md).
+- At 8B params (bf16 ~16GB), it's the smallest of the upgrade candidates —
+  which mattered when 6GB was the ceiling, but with 32GB available there's
+  no VRAM reason to pick the model the prior doc already flagged as "lower
+  fidelity than FLUX/SD3.5 by current standards" over Qwen-Image or
+  FLUX.1-dev. AMD's own ComfyUI-ROCm docs do list it as a first-party
+  example template ("SD3.5 Simple", per the prior doc), so it remains a fine
+  fallback if Qwen-Image's FP8 path hits the ROCm rough edge noted above.
+
+### HunyuanImage-3.0 — ruled out
+
+- Official repo states VRAM requirements of "≥ 3 × 80GB" for the base model
+  and "≥ 8 × 80GB" for HunyuanImage-3.0-Instruct — i.e. multi-GPU
+  datacenter-class NVIDIA clusters, not a single 32GB consumer/workstation
+  card at any precision. Source:
+  [Tencent-Hunyuan/HunyuanImage-3.0 GitHub repo](https://github.com/Tencent-Hunyuan/HunyuanImage-3.0).
+- Setup instructions require CUDA 12.8, PyTorch 2.8.0 built for CUDA, and
+  optionally FlashAttention2/FlashInfer for MoE routing speed — no AMD or
+  ROCm path is mentioned anywhere in the official repo. Even the
+  MoE-efficient "13B active" framing doesn't help here: the tooling itself
+  assumes an NVIDIA multi-GPU cluster, and the 80B total parameter set still
+  has to be resident somewhere.
+- 32GB of headroom on one AMD card doesn't move this model into reach at
+  any precision considered here; it's excluded regardless of how much VRAM
+  frees up on this specific box.
+
+### Krea-2 (Turbo) — promising, but unverified on this hardware
+
+- Verified directly against primary sources per the ticket's instruction
+  (this is a June 2026 release, past most training cutoffs): the
+  [Hugging Face model card](https://huggingface.co/krea/Krea-2-Turbo) states
+  a release date of **June 22, 2026**, a 12-billion-parameter single-stream
+  diffusion transformer, `torch.bfloat16` as the reference precision, and
+  the **Krea 2 Community License**.
+- License, per [krea.ai/krea-2-licensing](https://www.krea.ai/krea-2-licensing):
+  non-commercial (including explicitly personal/homelab) use is free;
+  commercial use is permitted royalty-free for entities under $1M
+  trailing-12-month revenue (same shape as SD3.5's and matching this
+  homelab's use case); content-filter and AI-disclosure obligations apply if
+  deployed publicly; derivative model names must start with "Krea".
+- ComfyUI added native support for both open-weight checkpoints (Krea 2 Raw
+  and Krea 2 Turbo) per [blog.comfy.org's announcement](https://blog.comfy.org/p/krea-2-open-source-models-are-now),
+  and community GGUF quants already exist
+  ([molbal/krea2-gguf](https://huggingface.co/molbal/krea2-gguf)), with
+  reports (RTX hardware only) of fp8 fitting 16GB and GGUF fitting 12GB.
+- **No AMD/ROCm mention anywhere** in the model card, and no gfx1201/R9700
+  community report was found — unsurprising given the model is roughly
+  2.5 months old at the time of this research. Its architecture (a
+  standard-shaped DiT that ComfyUI loads through its normal diffusion-model
+  nodes, per the ComfyUI blog post) gives reasonable expectation it will run
+  on the same ROCm/PyTorch backend already proven for FLUX and Qwen-Image on
+  this card, but that's an inference, not a verified fact the way
+  kyuz0's Qwen-Image workflow is.
+- **Not the pick today**, precisely because Qwen-Image already offers a
+  hardware-verified path at a comparable parameter count and VRAM budget.
+  Worth a follow-up research ticket once R9700/gfx1201-specific Krea-2
+  reports exist — the license and ComfyUI support are both already in
+  place, so the only open question is real-world ROCm behavior.
+
+## Recommendation
+
+**Qwen-Image, FP8 precision** (`qwen_image_fp8_e4m3fn.safetensors` +
+`qwen_2.5_vl_7b_fp8_scaled.safetensors`, ~25GB combined), optionally paired
+with the 4-step Lightning LoRA the R9700-specific validated workflow uses.
+It wins on all three axes the ticket asked about:
+
+1. **License**: Apache-2.0 — no restriction at all, strictly better than
+   every other candidate including the current FLUX.1-schnell pick.
+2. **ROCm/gfx1201 compatibility**: the only candidate with a workflow
+   pre-validated specifically on this GPU architecture
+   ([kyuz0/amd-r9700-comfy](https://github.com/kyuz0/amd-r9700-comfy)),
+   not just "ComfyUI supports ROCm in general."
+3. **VRAM at the ~32GB ceiling**: ~25GB at FP8 leaves real margin for
+   ComfyUI runtime/VAE overhead, without needing the multi-step,
+   non-distilled FLUX.1-dev's full 24GB bf16 footprint or accepting
+   SD3.5's lower fidelity ceiling — and it's a genuine capability upgrade
+   over schnell (20B vs. 12B, non-distilled-quality text rendering) rather
+   than just a bigger file.
+
+If the known Qwen-Image/ROCm edit-mode rough edge
+([pollockjj/ComfyUI-MultiGPU #133](https://github.com/pollockjj/ComfyUI-MultiGPU/discussions/133))
+turns out to affect plain text-to-image generation too, SD3.5-large (AMD's
+own first-party "SD3.5 Simple" ComfyUI-ROCm template workflow) is the
+fallback, with FLUX.1-dev as a second option. HunyuanImage-3.0 is excluded
+outright regardless of available VRAM (CUDA-only tooling, multi-GPU
+datacenter VRAM floor). Krea-2 is worth revisiting once R9700-specific
+field reports exist.
+
+## Sources consulted
+
+- [docs/research/image-generation-options.md](image-generation-options.md) (this repo — prior findings, not re-derived)
+- [Qwen/Qwen-Image (Hugging Face)](https://huggingface.co/Qwen/Qwen-Image)
+- [QwenLM/Qwen-Image (GitHub)](https://github.com/QwenLM/Qwen-Image)
+- [ComfyUI Wiki — Qwen-Image native support announcement](https://comfyui-wiki.com/en/news/2025-08-05-qwen-image)
+- [Comfy-Org/Qwen-Image_ComfyUI (Hugging Face, FP8 checkpoints)](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders)
+- [Comfy-Org/ComfyUI issue #10852 — Qwen-Image VRAM](https://github.com/Comfy-Org/ComfyUI/issues/10852)
+- [ComfyUI official Qwen-Image example](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/)
+- [kyuz0/amd-r9700-comfy (R9700-specific ROCm ComfyUI toolbox)](https://github.com/kyuz0/amd-r9700-comfy)
+- [pollockjj/ComfyUI-MultiGPU discussion #133 (Qwen-Image-Edit ROCm rough edge)](https://github.com/pollockjj/ComfyUI-MultiGPU/discussions/133)
+- [black-forest-labs/FLUX.1-dev (Hugging Face) + LICENSE.md](https://huggingface.co/black-forest-labs/FLUX.1-dev)
+- [city96/FLUX.1-dev-gguf](https://huggingface.co/city96/FLUX.1-dev-gguf)
+- [stabilityai/stable-diffusion-3.5-large (Hugging Face) + LICENSE.md](https://huggingface.co/stabilityai/stable-diffusion-3.5-large)
+- [Tencent-Hunyuan/HunyuanImage-3.0 (GitHub)](https://github.com/Tencent-Hunyuan/HunyuanImage-3.0)
+- [krea/Krea-2-Turbo (Hugging Face)](https://huggingface.co/krea/Krea-2-Turbo)
+- [Krea 2 Community License Agreement (krea.ai)](https://www.krea.ai/krea-2-licensing)
+- [blog.comfy.org — Krea 2 open-source models in ComfyUI](https://blog.comfy.org/p/krea-2-open-source-models-are-now)
+- [molbal/krea2-gguf (Hugging Face)](https://huggingface.co/molbal/krea2-gguf)
@@ -0,0 +1,249 @@
+# Research: adding local image generation to the stack
+
+**Question:** What's the best way to add local image generation alongside
+the existing Qwen3.8-27B / llama.cpp text stack, given a single AMD Radeon
+R9700 (32GB VRAM, ROCm/gfx1201 — not CUDA), routed through the OmniRoute
+gateway on the `ai-stack` Docker network?
+
+**Answer, short version:** run **ComfyUI** (official AMD-blessed ROCm
+Docker path exists, and OmniRoute already has a first-class `comfyui`
+provider — no bespoke API wrapper needed) with **FLUX.1 [schnell]**
+(Apache-2.0, 4-step, GGUF-quantizable) as the default model, falling back to
+**SDXL** for anything schnell's distilled-step license/quality tradeoffs
+don't suit. VRAM headroom against the current llama-server footprint is too
+tight for both to be resident at once at any real image quality — plan for
+**time-sliced use** (llama-server's existing lazytainer stop-on-idle pattern,
+mirrored for the image-gen service, or a manual "stop one, start the other"
+toggle), not concurrent operation.
+
+## Current VRAM baseline (from this repo)
+
+Per `docker-compose.yml` and `.env.example`, llama-server runs
+`Qwen3.8-27B-UD-Q4_K_XL.gguf` (17.6 GB weights) at `--ctx-size 262144` with
+`--cache-type-k q8_0 --cache-type-v q8_0`, landing at **~25.6 GB** total
+(weights + q8_0 KV cache), leaving **~6 GB** free on the 32GB card — this
+matches the math already recorded in
+[`docs/research/qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md). Per the auto-memory
+note on this repo, real measured VRAM use has run closer to ~75% (~24 GB) in
+practice versus the theoretical estimate, which doesn't change the
+conclusion below but means the ~6 GB figure is closer to a ceiling than a
+comfortable number.
+
+**Implication:** 6 GB is not enough for any current-generation image model at
+usable quality (see VRAM table below — even the smallest practical FLUX
+quant wants ~7 GB alone, before ComfyUI's own runtime/VAE overhead). Running
+image-gen *concurrently* with llama-server resident is not realistic on this
+card. The two need to time-share the GPU, not split it.
+
+## Backend evaluation (ROCm support, checked against primary sources)
+
+### ComfyUI — recommended
+
+- **Official AMD ROCm docs exist and are current.** AMD's own ROCm docs site
+  hosts a dedicated ComfyUI install guide with a prebuilt Docker image path
+  (recommended) or build-from-source, listing ROCm 7.2.0 and 7.1.0 as
+  supported versions, explicit `--device=/dev/kfd --device=/dev/dri
+  --group-add video` flags (same device-passthrough pattern this repo
+  already uses for llama-server), and template workflows including "SD3.5
+  Simple". Officially the guide only names AMD Instinct
+  MI355X/MI325X/MI300X (datacenter cards) as supported platforms.
+  Source: [ROCm docs — ComfyUI on ROCm installation](https://rocm.docs.amd.com/projects/comfyui/en/docs-26.04/install/comfyui-install.html).
+- **The upstream ComfyUI README itself documents AMD support directly**,
+  including consumer cards: stable ROCm install via
+  `pip install torch torchvision torchaudio --index-url
+  https://download.pytorch.org/whl/rocm7.2`, plus an experimental Windows
+  build explicitly naming **RDNA 3 (RX 7000), RDNA 3.5 (Strix Halo), and
+  RDNA 4 (RX 9000 series)** — i.e. the same RDNA4 generation as the R9700 —
+  and `HSA_OVERRIDE_GFX_VERSION` workarounds for older/unlisted cards.
+  Source: [comfyanonymous/ComfyUI README](https://github.com/comfyanonymous/ComfyUI).
+- **AMD has published a specific RDNA4/RX 9000 ComfyUI guide** (separate
+  from the Instinct-only install page above), confirming RDNA4 consumer
+  cards are an explicitly supported, first-party-documented target, not just
+  a community workaround.
+  Source: [ROCm blog — Getting Started with ComfyUI on AMD Radeon RX 9000 Series GPUs](https://rocm.blogs.amd.com/artificial-intelligence/comfyui-radeon-9000/README.html).
+- **gfx1201 (R9700's arch) specifically has active community Docker images**:
+  `yurisasc/comfyui-rocm-rdna4` targets ROCm 7.1 + PyTorch 2.9.1 with
+  `HSA_OVERRIDE_GFX_VERSION=12.0.1` / `PYTORCH_ROCM_ARCH=gfx1201` baked in,
+  and there's a published community patch specifically for R9700 AI Pro +
+  ComfyUI video-gen speedups, evidence the card is being run today, not just
+  theoretically compatible.
+  Sources: [yurisasc/comfyui-rocm-rdna4](https://github.com/yurisasc/comfyui-rocm-rdna4),
+  [charlie12345/R9700AIProComfyUIPatch](https://github.com/charlie12345/R9700AIProComfyUIPatch).
+- **Known gfx1201 caveat:** AMD's own TransformerEngine repo has an open
+  issue confirming gfx1201 is missing from the FP8 architecture table, so
+  FP8 kernels silently fall back to FP32 with ~50% throughput loss
+  (18-22 vs. 35-40 tok/s in the reporter's LLM benchmark) — not a
+  correctness blocker, but relevant if planning to use FP8-quantized image
+  models expecting native FP8 speed on this card; GGUF/Q-quants (see below)
+  avoid this path entirely since they dequantize to bf16/fp16, not fp8.
+  Source: [ROCm/TransformerEngine issue #520](https://github.com/ROCm/TransformerEngine/issues/520).
+- **Actively maintained community Docker packaging** beyond AMD's own image:
+  `YanWenKun/ComfyUI-Docker` ships parallel `rocm` (PyTorch-build-based,
+  faster releases) and `rocm7` (AMD-build-based, more comprehensive)
+  variants, both targeting ROCm 7, with ~1000 commits of ongoing history —
+  a viable alternative to the official AMD image if it lags behind ComfyUI
+  releases.
+  Source: [YanWenKun/ComfyUI-Docker](https://github.com/YanWenKun/ComfyUI-Docker).
+
+### AUTOMATIC1111 / Forge — usable but a step down for this hardware
+
+- ROCm support for A1111/Forge is real but community-patched, not
+  first-party. The upstream `lllyasviel/stable-diffusion-webui-forge` repo's
+  own discussion thread on AMD support points users to
+  `lshqqytiger/stable-diffusion-webui-amdgpu-forge`, a community fork
+  specifically maintained for AMD, "regarded as the go-to version" for
+  running FLUX-era models on AMD — i.e. the *mainline* Forge repo does not
+  claim ROCm support itself; you're expected to run a fork.
+  Source: [lllyasviel/stable-diffusion-webui-forge discussion #67](https://github.com/lllyasviel/stable-diffusion-webui-forge/discussions/67).
+- No first-party AMD vendor documentation (unlike ComfyUI's AMD-authored
+  ROCm/RDNA4 blog posts above) was found for A1111/Forge specifically.
+  Given ComfyUI already has an AMD-blessed path plus a first-class OmniRoute
+  provider (below), there's no reason to take on a community fork's
+  maintenance risk instead.
+
+### InvokeAI — usable but weaker AMD story for a new-generation card
+
+- InvokeAI documents ROCm support but flags it as second-tier: "AMD GPUs
+  are only supported on Linux," and "support for newer AMD GPUs is spotty
+  ... you may experience garbled images, black images, or long startup
+  delays." Its own install docs reference ROCm 5.4.2-era wheels, notably
+  older than the ROCm 7.x this stack's llama-server image already runs on
+  gfx1201.
+  Source: [InvokeAI installation docs (mauwii mirror)](https://mauwii.github.io/InvokeAI/installation/030_INSTALL_CUDA_AND_ROCM/).
+- No OpenAI-compatible-images angle either — same drawback as A1111/Forge.
+  Not recommended as primary given ComfyUI's stronger, more current AMD
+  documentation trail.
+
+## Model choice: FLUX.1 [schnell] vs FLUX.1 [dev] vs SDXL vs SD3.5
+
+| Model | License | Params | Notes |
+|---|---|---|---|
+| **FLUX.1 [schnell]** | **Apache-2.0** — fully open, no commercial restriction | 12B | Distilled for 1-4 step inference (fast); Black Forest Labs' own model card states this license directly |
+| FLUX.1 [dev] | [FLUX.1-dev Non-Commercial License](https://github.com/black-forest-labs/flux/blob/main/model_licenses/LICENSE-FLUX1-dev) | 12B | Non-commercial for the *model/weights*; generated *outputs* are explicitly usable commercially per BFL's license page. Higher quality than schnell (more steps, non-distilled) but the weights themselves can't be redistributed/used commercially |
+| SDXL | CreativeML OpenRAIL++ (permissive, commercial-friendly) | ~3.5B | Older (2023), lower fidelity than FLUX/SD3.5 by current standards, but lowest VRAM footprint and best long-standing tooling maturity |
+| SD3.5 (Large/Medium) | Stability AI Community License — free commercial use under $1M annual revenue, else enterprise license required | 8B / 2.5B | Free for this repo's non-commercial homelab use regardless; template already listed in AMD's own ComfyUI-ROCm doc ("SD3.5 Simple") as a first-party example workflow |
+
+Sources: [black-forest-labs/flux model cards](https://github.com/black-forest-labs/flux/blob/main/model_cards/FLUX.1-dev.md),
+[black-forest-labs/FLUX.1-schnell on Hugging Face](https://huggingface.co/black-forest-labs/FLUX.1-schnell)
+(license: apache-2.0), [FLUX.1-dev LICENSE.md](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md),
+[Stability AI — Introducing Stable Diffusion 3.5](https://stability.ai/news-updates/introducing-stable-diffusion-3-5),
+[stabilityai/stable-diffusion-3.5-large LICENSE.md](https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/LICENSE.md).
+
+**Recommendation: FLUX.1 [schnell].** For a private homelab, license
+enforcement isn't the deciding factor by itself, but schnell's Apache-2.0
+status removes any future ambiguity if outputs or the setup are ever shared
+or repurposed, and its whole design point — good quality in 1-4 sampling
+steps — directly addresses the VRAM/time-slicing constraint below (less
+GPU-resident time per image than a 20-50 step dev/SDXL/SD3.5 run).
+Quantized via `city96/ComfyUI-GGUF` (an actively-referenced, community-
+trusted quantization node — its GGUF Q-quants dequantize to bf16/fp16 at
+runtime, sidestepping the gfx1201 FP8 dequant bug above entirely), FLUX fits
+in a fraction of its fp16 footprint:
+
+| Precision | Approx. VRAM (model only) |
+|---|---|
+| fp16 (baseline) | ~24 GB |
+| fp8 | ~12 GB |
+| GGUF Q5_K_S | ~12-15 GB (practical quality floor) |
+| GGUF Q4_K_S | ~7 GB (quality starts degrading on hands/text below Q4) |
+
+Source: aggregated VRAM figures from GGUF-quantization write-ups referencing
+city96's FLUX GGUF conversions — treat as secondary/community sourced
+(no single BFL-published VRAM table was found), consistent across multiple
+independent sources.
+[city96/ComfyUI-GGUF README](https://github.com/city96/ComfyUI-GGUF/blob/main/README.md),
+[city96/FLUX.1-dev-gguf model card](https://huggingface.co/city96/FLUX.1-dev-gguf).
+
+**Fallback pick: SDXL.** If schnell's distilled quality ceiling proves too
+low for some use case, SDXL is the safer second choice over FLUX.1 [dev] or
+SD3.5 specifically *because* of this card's tight headroom: it's the
+smallest of the four by a wide margin, has the longest production track
+record on ROCm of any of these models, and its OpenRAIL++ license carries no
+revenue-threshold clause to track (unlike SD3.5's Community License) or
+non-commercial weight restriction (unlike FLUX.1 [dev]).
+
+## OpenAI-compatible API / OmniRoute integration
+
+This is the best news in this research: **OmniRoute already ships a
+first-class, built-in `comfyui` provider** — not a generic "point it at an
+OpenAI base URL and hope" integration. Its provider reference documents it
+explicitly: *"No API key required. Configure the local ComfyUI base URL
+(default: http://localhost:8188)."* OmniRoute's own image-routing feature
+set (`/v1/images/generations`, `/v1/images/edits`, `/v1/images/variations`,
+with automatic provider fallback) is designed for exactly this pattern:
+register ComfyUI as a backend, then any client already calling OmniRoute's
+OpenAI-compatible images endpoints reaches it with no extra shim.
+Source: [diegosouzapw/OmniRoute PROVIDER_REFERENCE.md](https://github.com/diegosouzapw/OmniRoute/blob/main/docs/reference/PROVIDER_REFERENCE.md),
+[diegosouzapw/OmniRoute repo description](https://github.com/diegosouzapw/OmniRoute).
+
+This means **ComfyUI does not need an extra OpenAI-API wrapper project** —
+the wrapper projects found during this research
+([`ComfyUI-OpenAI-Compatible-API`](https://github.com/yeeyou/ComfyUI-OpenAI-Compatible-API))
+turned out to be a ComfyUI *custom node* for calling *outbound* to LLM APIs
+from within a workflow (the reverse direction), not something this stack
+needs — OmniRoute's own native ComfyUI provider is the actual integration
+point, one layer up.
+
+**Confidence note:** the provider-reference detail above was fetched via an
+automated summarizer against the raw doc rather than manually re-verified
+line-by-line; re-check `PROVIDER_REFERENCE.md`'s `comfyui` entry directly
+before wiring this up, in case ComfyUI's own `/prompt` API (a
+workflow-graph-shaped API, not a simple text-prompt-in/image-out call) needs
+a specific default workflow JSON configured on the OmniRoute side to produce
+a plain text-to-image call.
+
+## Integration sketch (not a full compose — see caveats above)
+
+- New service in `docker-compose.yml`, e.g. `comfyui`, image
+  `rocm/comfyui-rocm` (or `yurisasc/comfyui-rocm-rdna4` for a gfx1201-tuned
+  build) or built from AMD's own ROCm ComfyUI Dockerfile, same
+  `/dev/kfd` + `/dev/dri` + `group_add: [video, render]` device-passthrough
+  block already used for `llama-server`, joined to the same `ai-stack`
+  network so `omniroute` can reach it as `http://comfyui:8188` — no host
+  port needed (matches the existing llama-server pattern of no published
+  port, gateway-only access).
+- Register it in OmniRoute's dashboard as a `comfyui` provider pointing at
+  that internal URL, same manual-registration pattern already used for
+  llama-server and searxng-search per `docs/proxy-key-onboarding.md`.
+- **VRAM contention is the real design problem, not networking.** Given the
+  ~6 GB headroom, the two services can't both sit GPU-resident.
+  Two workable patterns, in order of how well they fit what's already in
+  this repo:
+  1. **Mirror the existing lazytainer stop-on-idle pattern** already applied
+     to `llama-server` (`docker-compose.yml`'s `lazytainer.group.*` labels) —
+     add an equivalent idle-timeout group for `comfyui`, and rely on the two
+     services naturally not being hit at the same time for a single-user
+     homelab. This doesn't *guarantee* mutual exclusion (both could still be
+     woken concurrently and both try to fit in 6 GB free), so it's a
+     reasonable-effort fit, not a hard guarantee.
+  2. **Explicit mutual exclusion**: a small script/compose profile that
+     stops `llama-server` before starting `comfyui` (and vice versa) rather
+     than relying on lazytainer's independent idle timers — worth doing if
+     the reasonable-effort version above causes a visible OOM in practice.
+  Either way, this is a "pick one, then the other" story, not "run both."
+- Given FLUX.1 [schnell]'s 1-4 step design, a cold-start-and-generate cycle
+  (wake ComfyUI from lazytainer sleep, generate, let it idle back down) is
+  a reasonably good fit for occasional image requests through the same
+  gateway that already does this for llama-server.
+
+## Sources consulted
+
+- [ROCm docs — ComfyUI on ROCm installation](https://rocm.docs.amd.com/projects/comfyui/en/docs-26.04/install/comfyui-install.html)
+- [ROCm blog — ComfyUI on AMD Radeon RX 9000 Series (RDNA4)](https://rocm.blogs.amd.com/artificial-intelligence/comfyui-radeon-9000/README.html)
+- [comfyanonymous/ComfyUI README](https://github.com/comfyanonymous/ComfyUI)
+- [YanWenKun/ComfyUI-Docker](https://github.com/YanWenKun/ComfyUI-Docker)
+- [yurisasc/comfyui-rocm-rdna4](https://github.com/yurisasc/comfyui-rocm-rdna4)
+- [charlie12345/R9700AIProComfyUIPatch](https://github.com/charlie12345/R9700AIProComfyUIPatch)
+- [ROCm/TransformerEngine issue #520 (gfx1201 FP8 fallback)](https://github.com/ROCm/TransformerEngine/issues/520)
+- [lllyasviel/stable-diffusion-webui-forge discussion #67 (AMD support)](https://github.com/lllyasviel/stable-diffusion-webui-forge/discussions/67)
+- [InvokeAI CUDA/ROCm install docs](https://mauwii.github.io/InvokeAI/installation/030_INSTALL_CUDA_AND_ROCM/)
+- [black-forest-labs/flux GitHub repo + model cards](https://github.com/black-forest-labs/flux)
+- [black-forest-labs/FLUX.1-schnell (Hugging Face, Apache-2.0)](https://huggingface.co/black-forest-labs/FLUX.1-schnell)
+- [black-forest-labs/FLUX.1-dev LICENSE.md](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md)
+- [Stability AI — Introducing Stable Diffusion 3.5](https://stability.ai/news-updates/introducing-stable-diffusion-3-5)
+- [stabilityai/stable-diffusion-3.5-large LICENSE.md](https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/LICENSE.md)
+- [city96/ComfyUI-GGUF](https://github.com/city96/ComfyUI-GGUF)
+- [city96/FLUX.1-dev-gguf](https://huggingface.co/city96/FLUX.1-dev-gguf)
+- [diegosouzapw/OmniRoute](https://github.com/diegosouzapw/OmniRoute) and its `PROVIDER_REFERENCE.md`
+- [yeeyou/ComfyUI-OpenAI-Compatible-API](https://github.com/yeeyou/ComfyUI-OpenAI-Compatible-API) (checked and ruled out — wrong direction)
+- This repo: `docker-compose.yml`, `.env.example`, `docs/research/qwen3.8-27b-quant.md`
@@ -0,0 +1,194 @@
+# Research: Why lazytainer's idle-stop on llama-server doesn't fire, and what switch-model.sh should do about it
+
+**Question:** lazytainer is configured on `llama-server` (`docker-compose.yml`
+`lazytainer.group.llamaserver.*` labels) but its idle-stop never triggers in
+practice — OmniRoute appears to keep the container looking "active" to
+lazytainer's packet-threshold detector. Confirm the mechanism, find root
+cause, and recommend how the future `scripts/switch-model.sh` (#43, blocked)
+should handle GPU-residency swaps between `llama-server` and a new `comfyui`
+service given this.
+
+**Answer:** Confirmed. lazytainer's detector is a dumb per-port packet
+counter with no traffic classification — it cannot tell OmniRoute's
+background provider health-check pings apart from real inference traffic,
+and there is no config knob in lazytainer or a per-provider one in OmniRoute
+that fixes this. **`switch-model.sh` should bypass lazytainer entirely** for
+the swap: drive `docker compose stop`/`up -d` directly on both services,
+rather than trying to make lazytainer's idle-stop cooperate.
+
+## Current config (`docker-compose.yml`)
+
+```yaml
+labels:
+  - "lazytainer.group.llamaserver.sleepMethod=stop"
+  - "lazytainer.group.llamaserver.ports=8080"
+  - "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
+  - "lazytainer.group.llamaserver.minPacketThreshold=2"
+```
+
+`ports=8080` matches the container's real internal port (`expose: ["8080"]`,
+confirmed in the same file) — not a misconfiguration. `minPacketThreshold=2`
+is already far *below* lazytainer's own documented default of `30`, i.e. this
+deployment already tried loosening the threshold to make idle-stop easier to
+reach, not harder.
+
+## How lazytainer's detector actually works (primary source: `vmorganp/Lazytainer`)
+
+Confirmed against the project's README and Go source
+(`src/group.go`) on [github.com/vmorganp/Lazytainer](https://github.com/vmorganp/Lazytainer):
+
+- It captures packets with **gopacket/libpcap directly on the configured
+  `netInterface`** (default `eth0`), applying a BPF filter built from the
+  group's `ports` list (`"port 8080"` here, per the source's filter-string
+  construction, e.g. `"port 80 or port 81 or etc."` in the general case).
+- The filter matches **every packet to or from the port** — SYN, ACK,
+  data, FIN, everything. It is not restricted to new-connection SYNs.
+- Every `pollRate` seconds (default `30`; not overridden in this repo's
+  config) it samples a rolling packet counter (`rxHistory`) and compares the
+  delta against `minPacketThreshold`:
+  `rxHistory[0]+minPacketThreshold > rxHistory[len(rxHistory)-1]` → treated as
+  active, `inactiveSeconds` resets to 0.
+- `ignoreActiveClients` (default `false`, not set here) only changes whether
+  an ESTABLISHED-connection count is also checked; it does not add any
+  content- or source-based filtering.
+- **There is no mechanism anywhere in lazytainer to exclude specific traffic
+  (by source IP, path, header, or request type) from the packet count.** The
+  README's config table (`ports`, `inactiveTimeout`, `minPacketThreshold`,
+  `ignoreActiveClients`, `pollRate`, `sleepMethod`, `netInterface`) is
+  exhaustive — nothing else exists to tune this per-caller.
+
+Consequence: a single TCP connection to port 8080 — a bare connect + one
+small HTTP exchange + close — already produces well over `minPacketThreshold=2`
+packets purely from the handshake and teardown (SYN, SYN-ACK, ACK, ..., FIN,
+ACK), regardless of payload size or purpose. At this threshold, essentially
+*any* connection to the port counts as "active" and resets `inactiveTimeout`.
+Raising the threshold wouldn't help either — the fix would need to be
+"ignore packets from OmniRoute's health-checker," which the tool has no way
+to express; it only counts packets on a port, source-blind.
+
+## How OmniRoute actually touches registered providers (primary source: `diegosouzapw/OmniRoute`)
+
+Confirmed against
+[`docs/reference/ENVIRONMENT.md`](https://github.com/diegosouzapw/OmniRoute/blob/main/docs/reference/ENVIRONMENT.md)
+in the OmniRoute repo:
+
+- OmniRoute runs a **background credential/connection health-check
+  scheduler** (`src/lib/credentialHealth/scheduler.ts`) on
+  `CREDENTIAL_HEALTH_CHECK_INTERVAL`, default `300000` ms (5 min), minimum
+  `10000` ms (10s) — this periodically re-tests each registered provider's
+  connection, which for a provider like `llama-server` (a plain HTTP base
+  URL, no API key) means an actual request/connection to
+  `llama-server:8080`.
+- Results are cached for `CREDENTIAL_HEALTH_CACHE_TTL` (default also 5 min).
+- **Only one exclusion exists, and it's hardcoded by provider category, not
+  configurable per-provider**: search providers
+  (`SEARCH_VALIDATOR_CONFIGS` in
+  `src/lib/providers/validation/searchProviders.ts`, e.g. `tavily-search`)
+  are permanently skipped because their validation call is a real billed
+  upstream query. `llama-server` is an inference provider, not a search
+  provider — it is not in this exclusion list.
+- The only toggle that actually stops the sweep is global:
+  `OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK=1`/`true`, which "disable[s]
+  background periodic testing of provider connections" for **every**
+  registered provider at once. There is no documented per-provider
+  disable/pause flag in
+  [`docs/reference/PROVIDER_REFERENCE.md`](https://github.com/diegosouzapw/OmniRoute/blob/main/docs/reference/PROVIDER_REFERENCE.md) —
+  the dashboard's `/dashboard/providers` page is described only as where you
+  "enable, configure, and test each provider," with no documented
+  independent "pause health checks for this one provider" control.
+
+So: OmniRoute is not the sole cause, but it is a live, recurring cause. Every
+5 minutes (at most — could also be triggered ad hoc by dashboard/API use) it
+opens a connection to `llama-server:8080` purely to check the provider is
+alive, which is exactly the kind of traffic lazytainer's port-level counter
+cannot distinguish from real inference calls. With `inactiveTimeout=900`
+(15 min) and a health-check every ≤300s, the container practically always
+sees qualifying traffic before its idle timer would expire.
+
+## Root cause
+
+Two independent, both-true facts combine to defeat idle-stop:
+
+1. **lazytainer's detector is fundamentally traffic-blind** — it counts raw
+   packets on a port with no way to exclude any specific caller or traffic
+   class. This is a property of the tool, not a misconfiguration in this
+   repo (`ports=8080` is correct; `minPacketThreshold=2` is already at the
+   permissive end).
+2. **OmniRoute periodically pings every registered non-search provider**
+   (default every ≤5 min) to keep its health/availability status current,
+   and that ping is indistinguishable, at the packet level, from a real
+   inference request.
+
+Neither side offers a targeted fix: lazytainer has no allowlist/denylist by
+source, and OmniRoute's only "stop pinging" lever
+(`OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK`) is all-or-nothing across every
+provider, not scoped to just `llama-server`. Tuning `minPacketThreshold`
+higher or lower doesn't change the outcome either way, since the health-check
+traffic and real traffic land on the exact same port with no distinguishing
+packet-level signature.
+
+## Recommendation for `scripts/switch-model.sh` (#43)
+
+**Bypass lazytainer entirely for the GPU-residency swap.** Drive both
+services directly:
+
+```bash
+docker compose stop llama-server
+docker compose up -d comfyui
+# ...and the reverse when swapping back
+```
+
+Justification:
+
+- The swap is a **deliberate, scripted, known-in-advance** event — the
+  script always knows exactly which service should go up and which should
+  go down. Idle-stop detection exists to handle the case where nobody knows
+  when a service last had traffic; that's not this case, so routing the
+  swap through a passive heuristic (lazytainer's idle timer) that this
+  research shows is already unreliable for `llama-server` adds a point of
+  failure for no benefit. Direct `docker compose stop`/`up -d` is
+  deterministic and immune to the packet-counting confound described above.
+- Reconfiguring lazytainer's thresholds was considered and rejected: no
+  threshold value fixes a detector that cannot distinguish OmniRoute's
+  keepalive traffic from real traffic on the same port (see Root cause).
+  This is a ceiling in the tool itself, not a tuning problem.
+- Pausing OmniRoute's polling for the swap window was also considered.
+  It's the one lever available (`OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK`),
+  but it is global — it would blind OmniRoute's health status for *every*
+  provider (including `searxng-search`, if registered) for the duration of
+  the swap, and adds an extra env-toggle-and-restart step to the script for
+  a problem that direct compose control sidesteps completely. It's worth
+  flagging for #43's implementation as a *secondary* safety measure — briefly
+  disabling the sweep (or accepting that OmniRoute may show `llama-server` as
+  errored/offline for up to `CREDENTIAL_HEALTH_CHECK_INTERVAL` after it's
+  stopped) — but it should not be the primary mechanism the swap relies on.
+- This does **not** require removing the existing `lazytainer.group.llamaserver.*`
+  labels — they can stay for whatever idle-stop benefit they still provide
+  between swaps (e.g. genuinely idle periods where nothing, including
+  OmniRoute, has recently touched the container long enough to matter) while
+  `switch-model.sh` simply never depends on lazytainer to do the actual
+  stop/start for a swap.
+
+## Bottom line for #43 (blocked ticket, once unblocked)
+
+- `switch-model.sh` should call `docker compose stop <from-service>` /
+  `docker compose up -d <to-service>` directly — never rely on lazytainer's
+  idle-stop to free the GPU as part of a swap.
+- No lazytainer config change (threshold, ports, poll rate) is a viable fix;
+  the detector has no way to exclude OmniRoute's traffic by source.
+- Optionally, as a secondary hygiene step, the script may toggle
+  `OMNIROUTE_DISABLE_CREDENTIAL_HEALTH_CHECK` around the swap (or simply
+  tolerate a stale "errored" status in OmniRoute's dashboard for up to one
+  `CREDENTIAL_HEALTH_CHECK_INTERVAL`) to avoid OmniRoute flagging the
+  just-stopped provider as failed mid-swap — but this is cosmetic/status
+  hygiene, not what makes the swap itself work.
+
+Sources: [`vmorganp/Lazytainer`](https://github.com/vmorganp/Lazytainer)
+(README config table; `src/group.go` packet-capture and threshold-comparison
+logic), [`diegosouzapw/OmniRoute` —
+`docs/reference/ENVIRONMENT.md`](https://github.com/diegosouzapw/OmniRoute/blob/main/docs/reference/ENVIRONMENT.md)
+(credential health-check scheduler env vars), [`diegosouzapw/OmniRoute` —
+`docs/reference/PROVIDER_REFERENCE.md`](https://github.com/diegosouzapw/OmniRoute/blob/main/docs/reference/PROVIDER_REFERENCE.md)
+(provider dashboard controls), this repo's `docker-compose.yml`
+(`lazytainer.group.llamaserver.*` labels, `llama-server`/`omniroute` service
+definitions).
@@ -0,0 +1,330 @@
+# OmniRoute + Qwen Code CLI web search — setup research
+
+Investigates how to (a) confirm/complete OmniRoute's routing to this stack's local
+Qwen model, and (b) enable Qwen Code CLI's web-search tool, for a user running
+`qwen` from WSL against this repo's docker-compose stack.
+
+## What's already configured (verified live in WSL, 2026-09-05)
+
+Checked via `wsl.exe -- bash -lc '...'` against `~/.qwen/`:
+
+- **qwen-code CLI is installed**: `which qwen` → `/home/haylan/.local/bin/qwen`, `qwen --version` → `0.23.0`.
+- **`~/.qwen/settings.json` already points at this stack's OmniRoute gateway**, in the exact shape OmniRoute's own `setup-qwen` command produces (see below):
+  ```json
+  "modelProviders": {
+    "openai": [
+      {
+        "id": "qwen3.8-27b-local//models/Qwen3.8-27B-UD-Q4_K_XL.gguf",
+        "name": "qwen3.8-27b-local",
+        "envKey": "OMNIROUTE_API_KEY",
+        "baseUrl": "http://proxy-ai.home/v1",
+        "generationConfig": { "contextWindowSize": 131072 }
+      }
+    ]
+  },
+  "security": { "auth": { "selectedType": "openai" } },
+  "model": {
+    "name": "qwen3.8-27b-local//models/Qwen3.8-27B-UD-Q4_K_XL.gguf",
+    "baseUrl": "http://proxy-ai.home/v1"
+  }
+  ```
+  This targets `http://proxy-ai.home/v1` (this repo's OmniRoute gateway hostname per `docs/network-access.md`), reads the API key from the `OMNIROUTE_API_KEY` env var, and matches `docs/coding-cli-setup.md`'s convention of naming the registered provider `qwen3.8-27b-local`. Two backup files (`settings.json.bak-cbm-*`, `settings.json.save`) show earlier iterations of the same config — this was set up deliberately, not a stray default.
+  - **Not a gap — verified correct**: `contextWindowSize: 131072` matches `LLAMA_CTX_SIZE / LLAMA_PARALLEL` (`262144 / 2`), not half of it. `docker-compose.yml` (lines 21–22) runs llama-server with `--ctx-size ${LLAMA_CTX_SIZE:-262144} --parallel ${LLAMA_PARALLEL:-2}`, and `.env.example` (line 39) spells out that each of the two concurrent slots gets `LLAMA_CTX_SIZE / LLAMA_PARALLEL` tokens — i.e. 131072 per slot, matching commit `23e90fe` ("cap concurrent slots at 2"). So `~/.qwen/settings.json`'s value is correctly sized to what one slot actually offers; no fix needed here.
+  - **Unverified**: whether `OMNIROUTE_API_KEY` is actually set in the WSL environment or in a `~/.qwen/.env` file — `env | grep -i qwen` in the same session showed no `OMNIROUTE_API_KEY` in the *current* shell (only `PATH` entries matched `qwen`), and `~/.qwen/.env` wasn't checked (missed in the executed probe — see Open questions). If it's unset, `qwen` calls will fail auth against OmniRoute regardless of the `web_search` setup below.
+- **No web-search config exists yet**: `env | grep -i tavily` and `env | grep -i search` both returned nothing; `settings.json` has no `tools.webSearch` key and no `mcpServers` entry for Tavily/Bailian/GLM search or for OmniRoute's own MCP server (it does have an unrelated `mcpServers.codebase-memory-mcp` stdio entry for this repo's own codebase-memory tool). `grep -ril "tavily\|websearch\|web_search\|web-search" ~/.qwen` matched only unrelated project chat-log files (from an unrelated `shopware-420-seeds` project), not any config.
+- **Conclusion**: model routing (a) is already done. Web search (b) is not configured at all — no API key, no MCP server, no built-in-tool setting.
+
+## (a) OmniRoute → local Qwen model routing
+
+Sources: this repo's `docker-compose.yml` (lines 63–133) and `.env.example`
+(lines 54–90); `README.md` §"AI gateway (OmniRoute)"; `docs/coding-cli-setup.md`;
+OmniRoute's own docs at `github.com/mckazzy/OmniRoute-run-qwen`, ref
+`release/v3.8.50`.
+
+**Current repo state**: `docker-compose.yml`'s `omniroute` service comment (lines 63–67)
+states routing is registered "once through the dashboard or `POST /api/providers`
+after first boot, not checked into this repo." `OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS=true`
+and `OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS=true` are already set (lines 84–88) so the
+dashboard/API will accept `llama-server`'s container-internal address instead of
+rejecting it as a private URL.
+
+**Confirmed against OmniRoute's own reference docs**
+(`docs/reference/ENVIRONMENT.md` at the pinned ref):
+- `OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS` — default `false`; the doc says it is
+  **"REQUIRED for self-hosted providers"** (it names LM Studio, Ollama, vLLM,
+  Llamafile, Triton, SearXNG). Confirms the repo's own comment is correct and necessary.
+- `OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS` — default `true` ("local-first"); `false`
+  would block localhost/LAN/private ranges outright (cloud-metadata IPs stay
+  blocked either way).
+- `OMNIROUTE_WS_BRIDGE_SECRET` — "REQUIRED in production — when unset, all WS
+  bridge requests are rejected," generated via `openssl rand -base64 32` — matches
+  this repo's comment (lines 89–93) and `scripts/update.sh` autofill.
+
+`docs/reference/PROVIDER_REFERENCE.md` (same ref) lists **`llama-cpp`** as a
+built-in "Local, self-hosted" provider ID:
+
+> "Configure the OpenAI-compatible base URL (default: `http://127.0.0.1:8080/v1`)"
+
+This is a good match for this stack's `llama-server` container, which exposes
+port 8080 only on the internal `ai-stack` Docker network (`docker-compose.yml`
+lines 30–33, "No published host port"). Inside that network the service is
+reachable by its Compose service name, so the base URL to register should be
+`http://llama-server:8080/v1`, not `127.0.0.1` (127.0.0.1 inside the OmniRoute
+container would mean OmniRoute itself, not llama-server — they're different
+containers on the same bridge network).
+
+**Concrete steps** (dashboard, matching `docs/proxy-key-onboarding.md`'s
+existing pattern for reaching the dashboard):
+
+1. Reach the dashboard: from the R9700 box, `docker inspect -f
+   '{{.NetworkSettings.Networks.ai_stack.IPAddress}}' omniroute`, then browse
+   `http://<that-ip>:20128`; from elsewhere, SSH-tunnel
+   `ssh -L 20128:<container-ip>:20128 <host>` then browse `localhost:20128`.
+2. Providers → Add provider → **llama.cpp** (`llama-cpp` provider ID per
+   `PROVIDER_REFERENCE.md`).
+3. Set base URL to `http://llama-server:8080/v1` (the Compose service name — both
+   containers share the `ai-stack` network per `docker-compose.yml`'s `networks:
+   [ai-stack]` on both services). No API key needed (llama-server's endpoint is
+   unauthenticated internally, per `docs/network-access.md`).
+4. Register the model under that provider using the naming this repo already
+   assumes downstream (`qwen3.8-27b-local`, per `docs/coding-cli-setup.md` line 8)
+   — pick a model ID/name here and keep it consistent everywhere a CLI config
+   references it (`~/.qwen/settings.json`'s existing entry already assumes this name).
+5. Mint or reuse a virtual API key for the `qwen-code-cli` workload per
+   `docs/proxy-key-onboarding.md` (label `qwen-code-cli`), and confirm it's the
+   value behind `OMNIROUTE_API_KEY` in the WSL environment (or `~/.qwen/.env` —
+   see Open questions) that `~/.qwen/settings.json`'s `envKey` references.
+
+**OmniRoute's own automation for this exact CLI** — `docs/guides/CLI-INTEGRATIONS.md`
+at the pinned ref documents a dedicated `omniroute setup-qwen` command:
+
+> `omniroute setup-qwen --model qwen/qwen3.8-max-preview` — writes
+> `~/.qwen/settings.json` (V4 `modelProviders.openai` array) and stores
+> `OMNIROUTE_API_KEY` in `~/.qwen/.env`; supports `--yes` (non-interactive),
+> `--config-path` / `--env-path` (custom locations), and works in local or remote mode.
+
+The `~/.qwen/settings.json` found on this machine has exactly the V4
+`modelProviders.openai` shape this command produces, and the two `.bak`/`.save`
+files back that up — this was very likely already run once, pointed at whichever
+model ID was registered in the dashboard at the time (the `id` field embeds the
+GGUF filename, `qwen3.8-27b-local//models/Qwen3.8-27B-UD-Q4_K_XL.gguf`, matching
+`.env.example`'s `LLAMA_MODEL_FILE`). Re-running it after registering/renaming
+the provider in step 2–4 above is the fastest way to refresh this file if the
+registered model ID ever changes (`contextWindowSize: 131072` itself is already
+correct — see note above on `--parallel`).
+
+## (b) Qwen Code CLI web search
+
+Sources: `qwenlm.github.io/qwen-code-docs/en/developers/tools/web-search/`,
+`.../en/developers/tools/mcp-server/`, `.../en/users/configuration/settings/`;
+OmniRoute's `docs/frameworks/MCP-SERVER.md` and `docs/reference/PROVIDER_REFERENCE.md`
+at `release/v3.8.50`.
+
+**Qwen Code's web-search docs page states plainly**: the *original* built-in
+`web_search` tool ("Tavily/Google/GLM/DashScope multi-provider") **"and its
+configuration were removed."** Current options, per that same page:
+
+1. **New built-in `web_search` tool** — DashScope-only now, not multi-provider.
+   Needs `tools.webSearch.enabled: true` and `tools.webSearch.model` (e.g.
+   `"qwen3.6-plus"`) in `settings.json`, or equivalent env vars if `settings.json`
+   can't be edited; requires a `DASHSCOPE_API_KEY` (Alibaba Cloud). It "issues a
+   self-contained search request to a small auxiliary model with DashScope's
+   server-side `web_search` (and `web_extractor`) tools, and returns the
+   narrated findings plus source URLs" — i.e. it calls out to Alibaba's cloud,
+   not this stack's local model or SearXNG.
+   - **Caveat**: `users/configuration/settings/` (the canonical settings-schema
+     page) does **not** list `tools.webSearch` anywhere among its documented
+     `tools.*` keys — only `tools.sandbox`, `tools.shell`, `tools.core`,
+     `tools.exclude`, `tools.disabled`. This key may be genuinely undocumented
+     there, or newer than that page's last update. Treat `tools.webSearch` as
+     unconfirmed against the settings schema itself — verify with `qwen --help`
+     or by testing once a `DASHSCOPE_API_KEY` is available (see Open questions).
+2. **MCP-based search** — three named services: Alibaba Cloud Bailian WebSearch,
+   Tavily WebSearch, GLM WebSearch Prime — each added as an `mcpServers` entry
+   in `settings.json`. Confirmed schema from `developers/tools/mcp-server/`:
+   HTTP/SSE servers use `httpUrl` (or `url` for SSE) plus an optional `headers`
+   object, e.g.:
+   ```json
+   { "mcpServers": { "tavily": {
+       "httpUrl": "https://mcp.tavily.com/mcp/?tavilyApiKey=${TAVILY_API_KEY}"
+   } } }
+   ```
+   (stdio servers instead use `command`/`args`/`env`/`cwd`, as the existing
+   `codebase-memory-mcp` entry in this machine's `~/.qwen/settings.json` does.)
+
+**Neither of Qwen Code's own two paths uses this stack's existing SearXNG
+integration.** But OmniRoute — already in front of this stack's model — has its
+own MCP server with a **built-in multi-provider web-search tool**, and this
+repo already wires SearXNG through OmniRoute (`README.md` §"Web search":
+"The gateway also fronts SearXNG-backed web search"; `.env.example`'s
+`SEARXNG_LAN_IP` / `search.home` extra_hosts entry in `docker-compose.yml`
+lines 106–109). OmniRoute's `docs/frameworks/MCP-SERVER.md` (pinned ref):
+
+> "Web search through OmniRoute search gateway
+> (Serper/Brave/Perplexity/Exa/Tavily/Google PSE/Linkup/SearchAPI/SearXNG) with
+> failover" — exposed as an `omniroute_web_search` tool requiring the
+> `execute:search` scope.
+
+And `docs/reference/PROVIDER_REFERENCE.md` lists `searxng-search` as one of its
+12 built-in search-provider IDs: **"API key is optional. Set your SearXNG base
+URL. Some instances may require a bearer token for access."** — meaning
+SearXNG can be registered as a search provider in the OmniRoute dashboard the
+same way `llama-cpp` is registered as a model provider, no separate API key
+needed for a self-hosted SearXNG instance.
+
+**This means the path that reuses what's already deployed in this stack (SearXNG,
+already reachable from OmniRoute via `search.home`) is: connect qwen-code to
+OmniRoute's MCP server, not to Tavily/DashScope/GLM directly.** Concrete steps:
+
+1. In the OmniRoute dashboard, register SearXNG as a search provider
+   (`searxng-search`), pointing at `http://search.home` (already resolvable
+   inside the OmniRoute container via the `extra_hosts` entry in
+   `docker-compose.yml`). This may already be done — `README.md` implies the
+   gateway already fronts SearXNG-backed search, but confirm live in the
+   dashboard since, per the same `docker-compose.yml` comment (lines 63–67),
+   provider registration isn't checked into this repo.
+2. Mint an API key scoped for MCP search use — OmniRoute's `MCP-SERVER.md`
+   names `execute:search` (to actually call the search tool) and `mcp:connect`
+   (narrow, MCP-connect-only) as the relevant scopes; `manage`/`admin` also work
+   but are broader than needed.
+3. Add an `mcpServers` entry to `~/.qwen/settings.json` pointing at OmniRoute's
+   MCP endpoint, using the same `httpUrl`/`headers` shape Qwen Code already
+   supports for Tavily:
+   ```json
+   {
+     "mcpServers": {
+       "omniroute-search": {
+         "httpUrl": "http://proxy-ai.home/api/mcp/stream",
+         "headers": { "Authorization": "Bearer ${OMNIROUTE_SEARCH_KEY}" }
+       }
+     }
+   }
+   ```
+   (`proxy-ai.home` matches the hostname the model-provider entry already uses
+   in this same file; swap in whatever host:port actually fronts OmniRoute's API
+   port from WSL — `docs/network-access.md` says `proxy-ai.home` points at
+   `${OMNIROUTE_PORT:-4000}`, the *API* port, and `docker-compose.yml`/`.env.example`
+   separately track `OMNIROUTE_API_PORT` (default `20129`) as the
+   container-internal port — confirm which one NPM actually proxies to before
+   trusting the `/api/mcp/stream` path resolves through `proxy-ai.home` unchanged;
+   this wasn't independently verified against a live instance, see Open questions.)
+4. Set `OMNIROUTE_SEARCH_KEY` in the WSL shell profile (or in `~/.qwen/.env`,
+   consistent with how `setup-qwen` already stores `OMNIROUTE_API_KEY` there).
+5. Restart `qwen`; the model should now see an MCP tool for web search backed by
+   this stack's own SearXNG, routed and rate-limited the same way its LLM calls
+   already are.
+
+If instead the goal is simply "get *any* web search working fastest, reuse
+nothing," the plain Tavily-MCP or DashScope built-in-tool paths above are
+simpler (one API key, no dashboard provider registration) — but they bypass
+this stack's OmniRoute/SearXNG setup entirely and send queries to an external
+paid API instead.
+
+## Follow-up verification (2026-09-05, live checks)
+
+- **`OMNIROUTE_API_KEY` — confirmed set and working.** It's exported from
+  `~/.bashrc` (line 133), *not* `~/.qwen/.env` — invisible to a non-interactive
+  `bash -lc` probe because `.bashrc`'s standard top-of-file guard
+  (`case $- in *i*) ;; *) return;; esac`) skips the rest of the file for
+  non-interactive shells; a real interactive shell (`bash -ic`, or `wsl` +
+  `qwen` as actually run) sources it fine. Verified: `curl -H "Authorization:
+  Bearer $OMNIROUTE_API_KEY" http://proxy-ai.home/v1/models` → `200`. Routing (a)
+  is confirmed end-to-end, no action needed.
+- **MCP endpoint located from primary source** — fetched OmniRoute's
+  `docs/frameworks/MCP-SERVER.md` at `release/v3.8.50` directly. Resolves the
+  port ambiguity above: the MCP server runs on **port 20128** (dashboard/API
+  port), paths `/api/mcp/stream` (streamable HTTP), `/api/mcp/sse`, and
+  `/api/mcp/status`. It states: `/api/mcp/*` is in OmniRoute's `LOCAL_ONLY` authz
+  tier (`src/server/authz/routeGuard.ts`) — loopback-only by default; a
+  non-loopback client needs a key carrying the `manage` scope or the narrower
+  `mcp:connect` scope (added v3.8.0), and the server's Settings must have
+  `mcpEnabled` on with the matching `mcpTransport` selected. `omniroute_web_search`
+  itself additionally needs `execute:search`. No separate "MCP key type" exists —
+  same provider keys, different scopes.
+- **Live probe result**: `curl http://proxy-ai.home:20128/api/mcp/status` returns
+  `{"error":{"code":"AUTH_001","message":"Authentication required"}}` **identically
+  with or without** the `Authorization: Bearer $OMNIROUTE_API_KEY` header — the
+  existing model-routing key isn't recognized on this route at all, consistent
+  with it lacking `mcp:connect`/`manage`/`execute:search` scope and/or
+  `mcpEnabled` not yet being turned on in the dashboard. This is dashboard-side
+  state (not in git, no session credentials available from this environment) —
+  genuinely needs a human with dashboard access, not another probe.
+- **Config prepared** to unblock as soon as that's done: added an
+  `omniroute-search` entry to `~/.qwen/settings.json`'s `mcpServers` (backed up
+  the prior file first as `settings.json.bak-wayfinder-<timestamp>`):
+  ```json
+  "omniroute-search": {
+    "httpUrl": "http://proxy-ai.home:20128/api/mcp/stream",
+    "headers": { "Authorization": "Bearer ${OMNIROUTE_SEARCH_KEY}" }
+  }
+  ```
+  Deliberately a separate env var (`OMNIROUTE_SEARCH_KEY`), not reusing
+  `OMNIROUTE_API_KEY`, so the search-scoped key stays distinct from the
+  model-routing key — matches `docs/proxy-key-onboarding.md`'s per-workload
+  key pattern.
+
+## Resolution (2026-09-05, completed)
+
+The dashboard steps above turned out to need a different diagnosis than
+originally guessed — walked through live with a `oma_live_...` management
+token and a rotating set of `sk-...` provider keys the user supplied:
+
+- **`/api/providers` (management API) showed zero search providers at all**
+  — not a misconfigured `searxng-search` entry, it simply didn't exist as a
+  connection anymore (9 connections total, all LLM/chat providers). The
+  CHANGELOG at the pinned ref was checked and shows `/v1/search` under active
+  feature development (a `feat(search)` entry adding Firecrawl support), so
+  this wasn't an OmniRoute-side removal of the search system — the row was
+  just gone from this instance's own database (reason unconfirmed: update
+  migration vs. prior manual removal).
+- **Created it via the API**, not the dashboard UI — `POST /api/providers`
+  turned out to accept the same generic connection schema used for LLM
+  providers: `{"provider":"searxng-search","name":"searxng"}` (Zod-validated;
+  an empty-body POST surfaced the required fields). Then
+  `PATCH /api/providers/<id>` with `{"providerSpecificData":{"baseUrl":"http://search.home/search"}}`
+  set the real URL, replacing the catalog default.
+- **Verified end-to-end**: `POST /v1/search` with `provider: "searxng-search"`
+  returned real results (5 hits, `search_cost_usd: 0`, `cached: false`,
+  `response_time_ms: 4495`) — confirms `search.home`'s `extra_hosts` mapping
+  in `docker-compose.yml` resolves correctly from inside the OmniRoute
+  container and the whole chain (OmniRoute → SearXNG → results) works.
+- **`/api/mcp/status` confirmed `scopesEnforced: false`** on this instance —
+  the `mcp:connect`/`execute:search` scope requirement documented upstream
+  isn't actually being enforced here, so any valid provider key connects.
+  `mcpEnabled: true` already, transport `streamable-http`.
+- **Key rotation caveat hit live**: the first `sk-...` key the user shared
+  went from working to a flat 401 on *every* route (including `/v1/models`)
+  partway through testing — consistent with it having been revoked/rotated
+  server-side. A second key worked immediately. If this setup stops working
+  later, check for exactly this before re-diagnosing the whole chain.
+- **Final `~/.qwen/settings.json` `mcpServers` entry** (confirmed connected
+  via `qwen mcp list` → `✓ omniroute-search: ... - Connected`):
+  ```json
+  "omniroute-search": {
+    "httpUrl": "http://proxy-ai.home/api/mcp/stream",
+    "headers": { "Authorization": "Bearer ${OMNIROUTE_SEARCH_KEY}" }
+  }
+  ```
+  `OMNIROUTE_SEARCH_KEY` is exported in `~/.bashrc` alongside the existing
+  `OMNIROUTE_API_KEY`, holding the second (working) `sk-...` key.
+
+**Status: done.** `qwen` in WSL has a connected `omniroute-search` MCP server
+backed by this stack's own SearXNG instance — no external search API, no
+Alibaba DashScope dependency. Not yet exercised: an actual `qwen` chat turn
+that triggers the `omniroute_web_search` tool call (only the MCP handshake
+and the raw `/v1/search` call were verified directly).
+
+## Open questions / unverified
+
+- **`tools.webSearch.*` settings.json schema** — described on Qwen Code's
+  web-search doc page but absent from the canonical settings-schema page; not
+  independently confirmed (e.g. via `qwen --help` or source) — moot for this
+  setup since the MCP path (above) is what's being wired in, not the
+  DashScope-only built-in tool.
+- **DashScope vs SearXNG data-residency/cost tradeoffs** — out of scope here,
+  but worth noting the built-in `web_search` tool sends queries to Alibaba
+  Cloud regardless of this stack being otherwise fully self-hosted.
+- OmniRoute's own docs (already flagged in this repo's `README.md`) describe
+  stealth/anti-detection and TLS-interception features elsewhere in its repo;
+  none of that is exercised by anything in this note, but it's the same caveat
+  `README.md` already carries forward from issue #31.
@@ -0,0 +1,95 @@
+# Research: GPU pinned at 100% with two concurrent llama.cpp containers, and the intermittent "render" group startup error
+
+**Question:** After adding `llama-server-fast` (#44), real-hardware testing on the R9700
+showed `rocm-smi` pinned at 100% GPU / ~73-101W whenever both `llama-server` and
+`llama-server-fast` run concurrently, dropping to 3% / ~25-60W the moment either one
+alone is stopped. Separately, `docker compose up` intermittently failed with
+`Error response from daemon: unable to find group render: no matching entries in group file`
+— confirmed new since the second GPU service was added. See issue #5's comment thread
+for the raw `rocm-smi`/`free -h` output this doc is diagnosing.
+
+## GPU pin: root cause and fix
+
+**Confirmed via #5's own data**: either container alone is fine (3% GPU, low power).
+The pin only appears with two concurrent HIP-context-holding processes on the same
+GPU. This matches `ROCm/ROCm#5706` (already flagged as a risk in map #1) — full
+comment thread confirms:
+
+- Root cause: an AMD MES (Micro Engine Scheduler) firmware bug triggered by HIP
+  hardware-queue creation, pinning the GPU at boost clock the moment ROCm
+  initializes a queue. Not llama.cpp-specific — reproduced with vLLM and bare
+  PyTorch ROCm too. Source: [ROCm/ROCm#5706](https://github.com/ROCm/ROCm/issues/5706)
+  (`tcgu-amd`, AMD engineer, confirms MES firmware root cause; closed as
+  "fixed" in March, but a report as recent as May 24 shows it recurring even on
+  patched firmware/kernel).
+- **Validated workaround**: `GPU_MAX_HW_QUEUES=1` as a container env var. One
+  report ran a controlled before/after on the exact image this stack uses
+  (`ghcr.io/ggml-org/llama.cpp:server-rocm`, R9700/gfx1201):
+  baseline 100% GPU / 95W → with the var set, 3% GPU / 22W, VRAM unchanged.
+  Source: same thread, `interconnectedMe`'s comment.
+- **Semantics** (why this should apply to our two-container case, not just the
+  single-process case tested above): `GPU_MAX_HW_QUEUES` is a **per-process**
+  HIP runtime setting — it caps how many HSA/hardware queues *that process's*
+  HIP runtime allocates, default higher (over-subscription is what causes the
+  penalty). Source: [AMD ROCm workload-optimization docs](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html).
+  Since it's per-process, setting it on *each* container independently is the
+  correct scope — it should reduce total concurrent hardware-queue creation
+  across both processes, which is the trigger condition MES chokes on.
+  **Caveat**: no primary source explicitly tested two concurrent containers
+  both set to `GPU_MAX_HW_QUEUES=1` — this is a well-grounded extrapolation
+  from confirmed per-process semantics and the same root-cause mechanism, not
+  a directly-reproduced fix for our exact topology. Verify with `rocm-smi`
+  after applying, both containers up.
+
+## "unable to find group render" — a real Docker bug, not flaky hardware
+
+This is a known, documented Docker limitation, not something specific to this
+stack: `group_add` by **name** requires Docker to resolve that name against
+the **container's own** `/etc/group` file — if the image doesn't define a
+`render` entry there (common for minimal/slim base images), resolution fails.
+Source: [docker/cli#4714](https://github.com/docker/cli/issues/4714)
+("`docker run --group-add` by name doesn't add group from host as
+documented") and [docker/compose#7277](https://github.com/docker/compose/issues/7277)
+(same "no matching entries in group file" error).
+
+Confirms why it's now intermittent rather than always-broken: this repo's
+`docker-compose.yml` uses `group_add: [video, render]` (plain names) on
+**three** GPU services now (`llama-server`, `llama-server-fast`, `comfyui`).
+Docker Compose starts containers concurrently, and each does its own
+name-resolution lookup independently — with only one GPU service before #44,
+the resolution almost always won its race; with two (soon three, once
+`comfyui`'s downloader/model land per #46) the odds of losing that race and
+hitting the unresolved-name path go up. This is consistent with the user's
+own observation that it's new since the second GPU service.
+
+**Fix, already precedented in this repo**: `scripts/update.sh` already
+resolves the host's real `video`/`render` **numeric GIDs** for the `comfyui`
+service (`COMFYUI_VIDEO_GID`/`COMFYUI_RENDER_GID`, passed as app-level env
+vars) — but `comfyui`'s own `group_add:` still uses plain names too, so it
+isn't actually protected by that either. The correct fix per the Docker
+issues above: use the resolved **numeric GIDs** in `group_add:` itself
+(Compose accepts numeric strings directly), not names, on all three GPU
+services. Numeric GIDs skip the name-resolution step entirely, eliminating
+both the flakiness and the race.
+
+## Recommendation
+
+1. Add `GPU_MAX_HW_QUEUES=1` to both `llama-server` and `llama-server-fast`'s
+   `environment:` blocks. Verify with `rocm-smi` after redeploy, both
+   containers up — this is the one part of this doc that's extrapolated
+   rather than directly reproduced, so real confirmation matters here.
+2. Resolve host `video`/`render` GIDs once (generalize the existing
+   `COMFYUI_VIDEO_GID`/`COMFYUI_RENDER_GID` pattern in `scripts/update.sh`
+   to shared `HOST_VIDEO_GID`/`HOST_RENDER_GID` vars), and switch
+   `group_add:` on all three GPU services (`llama-server`,
+   `llama-server-fast`, `comfyui`) from `[video, render]` (names) to the
+   resolved numeric GIDs. Removes the race entirely rather than reducing its
+   odds.
+
+## Sources
+
+- [ROCm/ROCm#5706 — full comment thread](https://github.com/ROCm/ROCm/issues/5706)
+- [AMD ROCm — MI300/MI350 workload optimization docs (GPU_MAX_HW_QUEUES)](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html)
+- [docker/cli#4714 — group_add by name doesn't work as documented](https://github.com/docker/cli/issues/4714)
+- [docker/compose#7277 — "no matching entries in group file"](https://github.com/docker/compose/issues/7277)
+- This repo's issue #5 (real-hardware `rocm-smi`/`free -h` evidence this doc diagnoses)
@@ -1,89 +0,0 @@
-model_list:
-  - model_name: qwen3.8-27b-local
-    litellm_params:
-      # Static name — llama.cpp serves whatever model it loaded regardless of
-      # what's requested here; this string isn't shell-expanded (this file
-      # isn't docker-compose.yml, .env vars don't reach it).
-      model: openai/qwen3.8-27b-local
-      api_base: http://llama-server:8080/v1
-      api_key: local
-      # Qwen3 is a reasoning model — it spends output tokens on
-      # reasoning_content before ever writing content. Callers that don't
-      # set their own max_tokens (Open WebUI's default request didn't) hit
-      # llama.cpp's low default, so the model runs out mid-thought and
-      # content comes back empty. This is a floor, not a cap — any caller
-      # that passes its own max_tokens still overrides it.
-      # Raised from 4096: confirmed in the wild (llama-server logs) that
-      # 4096 wasn't enough — reasoning_content alone ate the whole budget on
-      # a real request (n_gen = 4096 exactly, no answer ever written). At
-      # ~26.7 t/s and a 65536-token context window, 16384 is a ~10-minute
-      # worst case, not the full ~20-minute worst case 32768 would be.
-      max_tokens: 16384
-    model_info:
-      # Shadow cloud-cost estimate — priced against Claude Sonnet 5's published
-      # rate, not real spend (this proxy only ever routes to the local model).
-      # Source: https://platform.claude.com/docs/en/about-claude/pricing,
-      # checked 2026-08-25. Update these two numbers if that page changes.
-      input_cost_per_token: 0.000002   # $2 / MTok
-      output_cost_per_token: 0.00001   # $10 / MTok
-
-  - model_name: local-embedding
-    litellm_params:
-      # Served by the dedicated embedding-server (nomic-embed-text-v1.5), not
-      # the chat model — see docker-compose.yml. Called by litellm-pgvector
-      # to embed knowledgebase content, and available directly at
-      # /v1/embeddings for anything else that wants it.
-      model: openai/local-embedding
-      api_base: http://embedding-server:8080/v1
-      api_key: local
-    model_info:
-      mode: embedding
-
-# SearXNG-backed web search — a standalone REST endpoint (/v1/search/searxng-search),
-# NOT a model-callable tool and not auto-injected into chat completions. See
-# docs/research/litellm-searxng-search.md. Requires the litellm container to
-# resolve search.home — see the `extra_hosts` entry in docker-compose.yml.
-search_tools:
-  - search_tool_name: searxng-search
-    litellm_params:
-      search_provider: searxng
-      api_base: http://search.home/
-
-# Knowledgebase / RAG, backed by the litellm-pgvector companion service (NOT
-# Qdrant — LiteLLM's native vector-store feature has no Qdrant provider, see
-# docs/research/litellm-knowledgebase.md). vector_store_id is this proxy's
-# own identifier for the store, not assigned by a backend.
-# Smoke-tested end-to-end against a running deploy (issue #24): search via
-# both /v1/vector_stores/{id}/search directly and the file_search tool on a
-# chat completion. Needed several fixes beyond this block to work — a
-# missing api_key here, litellm-pgvector's Prisma schema never having been
-# pushed, a 1536- vs 768-dim mismatch, and its create endpoint ignoring any
-# caller-supplied id — see scripts/update.sh, scripts/ingest-memory.sh, and
-# vendor/litellm-pgvector/'s local patches (models.py, main.py,
-# prisma/schema.prisma).
-vector_store_registry:
-  - vector_store_name: memory-and-notes
-    litellm_params:
-      vector_store_id: "memory-and-notes"
-      custom_llm_provider: pg_vector
-      api_base: http://litellm-pgvector:8000
-      # Required by litellm's pg_vector provider (see
-      # PGVectorStoreConfig.validate_environment in litellm's source) — it's
-      # the Bearer token litellm-pgvector's own API checks against its
-      # SERVER_API_KEY. Was missing entirely, which is why every vector
-      # store call was failing with "Incorrect API key provided: None"
-      # before litellm-pgvector was ever reached. See issue #24.
-      api_key: os.environ/LITELLM_PGVECTOR_API_KEY
-      embedding_model: local-embedding
-
-router_settings:
-  # ponytail: LiteLLM's request-prioritization scheduler is beta (see
-  # docs/proxy-request-priority.md) — exact settings key/shape must be
-  # confirmed against LiteLLM's current docs and smoke-tested against
-  # llama.cpp before workloads depend on it. Redis is available (see the
-  # litellm service's REDIS_* env vars in docker-compose.yml) if the
-  # scheduler needs shared state for it.
-  enable_priority_scheduling: true
-
-general_settings:
-  master_key: os.environ/LITELLM_MASTER_KEY
@@ -1,85 +0,0 @@
-#!/usr/bin/env bash
-# Loads data/memory.md and data/claude-legacy-memory.md into the LiteLLM
-# knowledgebase (the "memory-and-notes" vector store, see litellm-config.yaml)
-# via litellm-pgvector's batch-embeddings endpoint. Both files are optional —
-# a file that doesn't exist yet is skipped, not an error.
-#
-# ponytail: one chunk per non-empty, non-heading line — both source files are
-# already one fact/paragraph per line (no hard-wrapping), so this needs no
-# real chunking logic. Re-run after editing either file; there's no dedup, so
-# this appends duplicates on a second run against unchanged content — clear
-# the store first (DELETE the vector_store_id's rows) if you need a clean
-# reload.
-set -euo pipefail
-cd "$(dirname "$0")/.."
-
-[ -f .env ] && set -a && . ./.env && set +a
-
-: "${LITELLM_PGVECTOR_API_KEY:?Set LITELLM_PGVECTOR_API_KEY in .env first}"
-: "${LITELLM_PGVECTOR_EMBEDDING_KEY:?Set LITELLM_PGVECTOR_EMBEDDING_KEY in .env first}"
-LITELLM_PGVECTOR_URL="${LITELLM_PGVECTOR_URL:-http://localhost:8000}"
-LITELLM_URL="${LITELLM_URL:-http://localhost:${LITELLM_PORT:-4000}}"
-VECTOR_STORE_ID="memory-and-notes"
-
-# Must match litellm-config.yaml's vector_store_registry entry — the
-# registry just points at a store the backend must already know about.
-# Ignores failure if it already exists (no documented idempotency check).
-# id is a local addition to litellm-pgvector's create endpoint (see
-# vendor/litellm-pgvector/main.py) — without it, create always minted a
-# random UUID and this script's writes could never land on VECTOR_STORE_ID.
-curl -sf -X POST "${LITELLM_PGVECTOR_URL}/v1/vector_stores" \
-  -H "Authorization: Bearer ${LITELLM_PGVECTOR_API_KEY}" \
-  -H "Content-Type: application/json" \
-  -d "{\"id\": \"${VECTOR_STORE_ID}\", \"name\": \"${VECTOR_STORE_ID}\"}" > /dev/null 2>&1 || true
-
-ingest_file() {
-  local file="$1"
-  if [ ! -f "$file" ]; then
-    echo "Skipping $file (not present)."
-    return
-  fi
-
-  local section="" contents="[]" metas="[]"
-  while IFS= read -r line; do
-    case "$line" in
-      "#"*) section="${line#\# }"; section="${section#\#\# }"; continue ;;
-      ""|"---") continue ;;
-    esac
-    contents=$(jq --arg c "$line" '. += [$c]' <<<"$contents")
-    metas=$(jq --arg content "$line" --arg source "$file" --arg section "$section" \
-      '. += [{"content": $content, "metadata": {"source": $source, "section": $section}}]' <<<"$metas")
-  done < "$file"
-
-  local n
-  n=$(jq 'length' <<<"$contents")
-  if [ "$n" -eq 0 ]; then
-    echo "Nothing to ingest from $file (no fact/paragraph lines)."
-    return
-  fi
-
-  # litellm-pgvector's embeddings endpoints take a precomputed vector per
-  # chunk — they don't call the embedding model themselves (only query-time
-  # search does, via its own EMBEDDING__* config). So this has to embed
-  # client-side first, via the same proxy every other workload uses.
-  echo "Embedding $n chunks from $file via LiteLLM..."
-  local embeddings
-  embeddings=$(curl -sf "${LITELLM_URL}/v1/embeddings" \
-    -H "Authorization: Bearer ${LITELLM_PGVECTOR_EMBEDDING_KEY}" \
-    -H "Content-Type: application/json" \
-    -d "$(jq -n --argjson input "$contents" '{"model": "local-embedding", "input": $input}')" \
-    | jq '[.data[].embedding]')
-
-  local batch
-  batch=$(jq -n --argjson metas "$metas" --argjson embeds "$embeddings" \
-    '[range(0; ($metas | length)) as $i | $metas[$i] + {"embedding": $embeds[$i]}]')
-
-  echo "Ingesting $n chunks from $file..."
-  curl -sf -X POST "${LITELLM_PGVECTOR_URL}/v1/vector_stores/${VECTOR_STORE_ID}/embeddings/batch" \
-    -H "Authorization: Bearer ${LITELLM_PGVECTOR_API_KEY}" \
-    -H "Content-Type: application/json" \
-    -d "$(jq -n --argjson embeddings "$batch" '{"embeddings": $embeddings}')" > /dev/null
-}
-
-ingest_file data/memory.md
-ingest_file data/claude-legacy-memory.md
-echo "Done."
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Swap GPU residency between llama-server (Qwen) and comfyui — they never
+# run concurrently, VRAM doesn't fit both (see issue #38's map). Manual
+# invocation only, no auto-switching.
+#
+# Bypasses lazytainer entirely and drives docker compose directly — its
+# idle-stop can't be used for this. Root cause (see
+# docs/research/lazytainer-omniroute-idle-stop.md, issue #40): lazytainer's
+# packet-threshold detector is source-blind and can't tell OmniRoute's
+# periodic health-check pings apart from real traffic on the same port, so
+# it never reliably sleeps a service on its own. A scripted swap always
+# knows which service should go up/down, so it doesn't need that heuristic.
+#
+# llama-server-fast (the small classifier model, issue #44) is NOT part of
+# this swap — it's meant to stay always-resident. Worst case with comfyui up
+# is comfyui (~25GB, Qwen-Image FP8) + llama-server-fast (~5GB) ≈ 30GB,
+# still under the 32GB card but tight — unverified on real hardware, check
+# `docker compose ps` / VRAM usage after the first real swap.
+#
+# OmniRoute may show the just-stopped provider as errored/offline in its
+# dashboard for up to CREDENTIAL_HEALTH_CHECK_INTERVAL (default 5 min) after
+# a swap — cosmetic, not a functional problem (see the research doc above).
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+usage() {
+  echo "Usage: $0 {qwen|comfyui}" >&2
+  echo "  qwen    - stop comfyui, start llama-server" >&2
+  echo "  comfyui - stop llama-server, start comfyui" >&2
+  exit 1
+}
+
+[ $# -eq 1 ] || usage
+
+case "$1" in
+  qwen)
+    from=comfyui
+    to=llama-server
+    ;;
+  comfyui)
+    from=llama-server
+    to=comfyui
+    ;;
+  *)
+    usage
+    ;;
+esac
+
+echo "==> stopping $from"
+docker compose stop "$from"
+
+echo "==> starting $to"
+docker compose up -d "$to"
+
+echo "==> status"
+docker compose ps
@@ -1,9 +1,24 @@
 #!/usr/bin/env bash
-# The one command to run after any change to this repo (compose file,
-# litellm-config.yaml, .env, or a git pull) to bring the running stack in
-# sync. Ensures secrets/keys exist, pulls, validates, rebuilds/re-pulls
-# images, and recreates only what changed — safe to run any time, including
-# with nothing to do.
+# The one command to run after any change to this repo (compose file, .env,
+# or a git pull) to bring the running stack in sync. Ensures secrets/keys
+# exist, pulls, validates, rebuilds/re-pulls images, and recreates only what
+# changed — safe to run any time, including with nothing to do.
+#
+# Tunable config values (LLAMA_*, ports, timeouts — anything with a real
+# default in .env.example) are synced from .env.example every run. A value
+# already matching is left alone silently. A value that DIFFERS from the
+# server's current .env is a conflict: interactively, you're shown every
+# conflict on one screen (via gum) and choose which to accept — unpicked
+# keys keep the server's current value. Non-interactively (no TTY — cron,
+# CI, piped), any conflict is a hard error unless --force is passed, which
+# accepts every new value automatically. Secrets and host-resolved values
+# (blank in .env.example — OMNIROUTE_*_SECRET/_KEY/_SALT/_PASSWORD,
+# SEARXNG_LAN_IP, COMFYUI_PUID/PGID, HOST_VIDEO_GID/RENDER_GID) are never
+# touched by this — they keep going through set_if_blank as before.
+#
+# omniroute's own routing/provider config (llama-server, search) lives in
+# its dashboard, not a checked-in file like the old litellm-config.yaml —
+# see issue #31 and docs/proxy-key-onboarding.md.
 #
 # ponytail: no rollback/backup logic — this is a single-user homelab box,
 # not a fleet. If a bad config lands, `git revert` + re-run is the recovery
@@ -11,8 +26,135 @@
 set -euo pipefail
 cd "$(dirname "$0")/.."

+FORCE=false
+for arg in "$@"; do
+  case "$arg" in
+    --force) FORCE=true ;;
+    *) echo "Usage: $0 [--force]" >&2; exit 1 ;;
+  esac
+done
+
+# Must run before anything else touches a file this script itself reads
+# (docker-compose.yml, .env.example, this script's own remaining lines) —
+# a self-updating script isn't guaranteed atomic against its own file
+# changing mid-run, so pulling later can execute a mix of old and new
+# script/compose content in one pass. Bit us for real: old GID-resolution
+# code ran, then this pulled in new var names docker-compose.yml now
+# requires, and nothing re-ran the (now-current) resolution step for
+# them — see issue #5's thread.
+echo "==> git pull"
+git pull --ff-only
+
 [ -f .env ] || cp .env.example .env

+echo "==> syncing tracked config values from .env.example"
+# ponytail: gum (charmbracelet/gum) is a single static binary. Vendored as
+# a release tarball under scripts/vendor/ (checked into git) for the R9700
+# box, which has no outbound internet access — the download fallback below
+# is only for other archs / when the vendored copy is missing or stale.
+# Cached under .cache/gum/ (gitignored) so repeat runs don't re-extract.
+GUM_VERSION="0.14.5"
+GUM_DIR="$(pwd)/.cache/gum"
+GUM_BIN="$GUM_DIR/gum"
+ensure_gum() {
+  command -v gum >/dev/null 2>&1 && { echo "gum"; return; }
+  [ -x "$GUM_BIN" ] && { echo "$GUM_BIN"; return; }
+  mkdir -p "$GUM_DIR"
+  local arch tmpdir vendored
+  case "$(uname -m)" in
+    x86_64) arch="x86_64" ;;
+    aarch64|arm64) arch="arm64" ;;
+    *) echo "no gum build for $(uname -m), falling back to plain prompts" >&2; echo ""; return ;;
+  esac
+  tmpdir="$(mktemp -d)"
+  vendored="$(pwd)/scripts/vendor/gum_${GUM_VERSION}_Linux_${arch}.tar.gz"
+  if [ -f "$vendored" ]; then
+    tar -xz -C "$tmpdir" -f "$vendored"
+  else
+    local url="https://github.com/charmbracelet/gum/releases/download/v${GUM_VERSION}/gum_${GUM_VERSION}_Linux_${arch}.tar.gz"
+    if ! curl -fsSL "$url" | tar -xz -C "$tmpdir" 2>/dev/null; then
+      echo "no vendored gum for $arch and couldn't download from $url (no internet egress? falling back to plain prompts)" >&2
+    fi
+  fi
+  if [ -n "$(find "$tmpdir" -name gum -type f 2>/dev/null)" ]; then
+    find "$tmpdir" -name gum -type f -exec cp {} "$GUM_BIN" \;
+    chmod +x "$GUM_BIN" 2>/dev/null || true
+  fi
+  rm -rf "$tmpdir"
+  [ -x "$GUM_BIN" ] && echo "$GUM_BIN" || echo ""
+}
+
+# Collect every key where .env.example has a real (non-blank) default:
+# missing from .env -> just add it (no conflict, nothing to decide);
+# present and identical -> leave alone silently; present and different ->
+# a conflict to resolve below.
+conflict_keys=()
+conflict_old=()
+conflict_new=()
+while IFS='=' read -r key value; do
+  [ -n "$value" ] || continue
+  if ! grep -qE "^${key}=" .env; then
+    echo "${key}=${value}" >> .env
+    continue
+  fi
+  current="$(grep -E "^${key}=" .env | head -1 | cut -d= -f2-)"
+  if [ "$current" != "$value" ]; then
+    conflict_keys+=("$key")
+    conflict_old+=("$current")
+    conflict_new+=("$value")
+  fi
+done < <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=.+' .env.example)
+
+if [ "${#conflict_keys[@]}" -gt 0 ]; then
+  if [ "$FORCE" = true ]; then
+    for i in "${!conflict_keys[@]}"; do
+      key="${conflict_keys[$i]}"; new="${conflict_new[$i]}"
+      sed -i "s|^${key}=.*|${key}=${new}|" .env
+      echo "${key}: ${conflict_old[$i]} -> ${new} (--force)"
+    done
+  elif [ ! -t 0 ] || [ ! -t 1 ]; then
+    echo "ERROR: ${#conflict_keys[@]} config value(s) in .env differ from .env.example, and this isn't an interactive terminal:" >&2
+    for i in "${!conflict_keys[@]}"; do
+      echo "  ${conflict_keys[$i]}: ${conflict_old[$i]} (current) vs ${conflict_new[$i]} (.env.example)" >&2
+    done
+    echo "Re-run interactively to choose per-key, or pass --force to accept every new value." >&2
+    exit 1
+  else
+    gum_bin="$(ensure_gum)"
+    labels=()
+    for i in "${!conflict_keys[@]}"; do
+      labels+=("${conflict_keys[$i]}: ${conflict_old[$i]} -> ${conflict_new[$i]}")
+    done
+    if [ -n "$gum_bin" ]; then
+      selected="$(printf '%s\n' "${labels[@]}" | "$gum_bin" choose --no-limit --selected "$(printf '%s\n' "${labels[@]}" | paste -sd,)" --header "Config differs from .env.example — selected keys take the new value, unselected keep the server's current value:")"
+    else
+      # ponytail: plain-bash fallback if gum couldn't be fetched (offline,
+      # unsupported arch) — same one-screen-of-conflicts idea, cruder UI.
+      echo "Config differs from .env.example. Enter space-separated numbers to KEEP the server's current value (all others take the new value), or press enter to take every new value:"
+      for i in "${!conflict_keys[@]}"; do
+        echo "  $((i+1))) ${labels[$i]}"
+      done
+      read -r -p "> " keep_nums
+      selected=""
+      for i in "${!conflict_keys[@]}"; do
+        case " $keep_nums " in
+          *" $((i+1)) "*) ;;
+          *) selected="${selected}${labels[$i]}"$'\n' ;;
+        esac
+      done
+    fi
+    for i in "${!conflict_keys[@]}"; do
+      key="${conflict_keys[$i]}"; new="${conflict_new[$i]}"
+      if printf '%s\n' "$selected" | grep -qxF "${labels[$i]}"; then
+        sed -i "s|^${key}=.*|${key}=${new}|" .env
+        echo "${key}: ${conflict_old[$i]} -> ${new}"
+      else
+        echo "${key}: kept ${conflict_old[$i]} (server value)"
+      fi
+    done
+  fi
+fi
+
 # Handles all three cases: the KEY=value line is missing entirely (.env
 # predates that var being added to .env.example — sed can't fix what isn't
 # there, so this appends it), present but blank, or already set.
@@ -31,19 +173,22 @@ set_if_blank() {

 echo "==> filling in missing secrets"
 # Random values — safe to re-run, never overwrites what's already set.
-# LITELLM_SALT_KEY especially: never change it after first run, existing
-# encrypted data becomes unreadable if you do.
-set_if_blank LITELLM_MASTER_KEY "$(openssl rand -hex 32)"
-set_if_blank LITELLM_SALT_KEY "$(openssl rand -hex 32)"
-set_if_blank LITELLM_DB_PASSWORD "$(openssl rand -hex 32)"
-set_if_blank REDIS_PASSWORD "$(openssl rand -hex 32)"
-set_if_blank UI_PASSWORD "$(openssl rand -hex 16)"
-set_if_blank PGVECTOR_DB_PASSWORD "$(openssl rand -hex 32)"
-set_if_blank LITELLM_PGVECTOR_API_KEY "$(openssl rand -hex 32)"
+# OMNIROUTE_STORAGE_ENCRYPTION_KEY especially: never change it after first
+# run, existing encrypted data becomes unreadable if you do (same caveat as
+# LiteLLM's old LITELLM_SALT_KEY).
+set_if_blank OMNIROUTE_INITIAL_PASSWORD "$(openssl rand -hex 16)"
+set_if_blank OMNIROUTE_JWT_SECRET "$(openssl rand -base64 48)"
+set_if_blank OMNIROUTE_API_KEY_SECRET "$(openssl rand -hex 32)"
+set_if_blank OMNIROUTE_STORAGE_ENCRYPTION_KEY "$(openssl rand -hex 32)"
+set_if_blank OMNIROUTE_MACHINE_ID_SALT "$(openssl rand -hex 16)"
+set_if_blank OMNIROUTE_CLI_SALT "$(openssl rand -hex 16)"
+set_if_blank OMNIROUTE_WS_BRIDGE_SECRET "$(openssl rand -hex 32)"
+set_if_blank NEO4J_PASSWORD "$(openssl rand -hex 16)"

 echo "==> resolving SEARXNG_LAN_IP"
 # search.home is a LAN mDNS/local-DNS name — resolvable from this host, just
-# not from inside the litellm container (see docs/research/litellm-searxng-search.md).
+# not from inside the omniroute container (see docs/research/litellm-searxng-search.md,
+# still the relevant background even though omniroute replaced litellm — see issue #31).
 searxng_ip="$(getent hosts search.home 2>/dev/null | awk '{print $1}' | head -1)"
 if [ -n "$searxng_ip" ]; then
  set_if_blank SEARXNG_LAN_IP "$searxng_ip"
@@ -51,8 +196,30 @@ else
  echo "SEARXNG_LAN_IP: couldn't resolve search.home from this host, set it manually if still blank."
 fi

-echo "==> git pull"
-git pull --ff-only
+echo "==> resolving ComfyUI host UID"
+# yurisasc/comfyui-rocm7.1 wants these as env vars, not just group_add in
+# compose — resolve from this host, same pattern as SEARXNG_LAN_IP.
+set_if_blank COMFYUI_PUID "$(id -u)"
+set_if_blank COMFYUI_PGID "$(id -g)"
+
+echo "==> resolving host video/render GIDs (shared by every GPU service)"
+# Numeric GIDs, not names, in docker-compose.yml's group_add: — Docker
+# resolves a *named* group_add entry against the container's own /etc/group,
+# not the host's, and fails unpredictably (worse with multiple GPU services
+# starting concurrently and racing on the same lookup) — see
+# docs/research/rocm-gpu-pin-and-render-group.md and issue #5.
+video_gid="$(getent group video 2>/dev/null | cut -d: -f3)"
+render_gid="$(getent group render 2>/dev/null | cut -d: -f3)"
+if [ -n "$video_gid" ]; then
+  set_if_blank HOST_VIDEO_GID "$video_gid"
+else
+  echo "HOST_VIDEO_GID: no 'video' group on this host, set it manually if still blank."
+fi
+if [ -n "$render_gid" ]; then
+  set_if_blank HOST_RENDER_GID "$render_gid"
+else
+  echo "HOST_RENDER_GID: no 'render' group on this host, set it manually if still blank."
+fi

 echo "==> validating compose config"
 docker compose config -q
@@ -65,56 +232,22 @@ docker compose build --pull

 echo "==> ensuring models are downloaded (skips already-present files)"
 docker compose --profile tools run --rm downloader
-docker compose --profile tools run --rm downloader-embedding
+docker compose --profile tools run --rm downloader-fast
+docker compose --profile tools run --rm downloader-comfyui

-echo "==> bringing up litellm (needed to mint virtual keys below)"
-docker compose up -d --wait litellm-db litellm
+echo "==> bringing up omniroute"
+docker compose up -d --wait omniroute

-# OPENWEBUI_LITELLM_KEY / LITELLM_PGVECTOR_EMBEDDING_KEY are per-workload
-# virtual keys, not random secrets — minted via LiteLLM's own API
-# (docs/proxy-key-onboarding.md documents the manual Admin UI route; this is
-# the same thing over the REST endpoint LITELLM_MASTER_KEY already
-# authenticates against).
-set -a && . ./.env && set +a
-mint_key_if_blank() {
-  local key="$1" alias="$2"
-  if grep -qE "^${key}=.*[^[:space:]]" .env; then
-    echo "${key}: already set, skipping."
-    return
-  fi
-  local minted
-  minted=$(curl -sf -X POST "http://localhost:${LITELLM_PORT:-4000}/key/generate" \
-    -H "Authorization: Bearer ${LITELLM_MASTER_KEY}" \
-    -H "Content-Type: application/json" \
-    -d "{\"key_alias\": \"${alias}\"}" | jq -r '.key')
-  if [ -n "$minted" ] && [ "$minted" != "null" ]; then
-    # Same missing-line-vs-blank-line handling as set_if_blank above.
-    if grep -qE "^${key}=" .env; then
-      sed -i "s|^${key}=.*|${key}=${minted}|" .env
-    else
-      echo "${key}=${minted}" >> .env
-    fi
-    echo "${key}: minted."
-  else
-    echo "${key}: mint failed, create it by hand per docs/proxy-key-onboarding.md."
-  fi
-}
-mint_key_if_blank OPENWEBUI_LITELLM_KEY openwebui
-mint_key_if_blank LITELLM_PGVECTOR_EMBEDDING_KEY litellm-pgvector
-set -a && . ./.env && set +a
+# ponytail: no scripted key-minting yet, unlike the old LiteLLM /key/generate
+# flow — omniroute's POST /api/keys needs a dashboard login session
+# (ManagementSessionAuth), not a static bearer key, and that flow hasn't
+# been verified against a live instance (see issue #37). No in-stack
+# workload needs a key right now (nothing left calls the gateway besides
+# coding CLIs, which mint their own by hand per docs/proxy-key-onboarding.md)
+# — revisit this script once that flow is automatable.

 echo "==> recreating changed services"
 docker compose up -d --remove-orphans

-# litellm-pgvector's Dockerfile only runs `prisma generate` (codegen) at
-# build time — nothing ever applied the schema to pgvector-db itself, so the
-# vector_stores/embeddings tables plain didn't exist until this was added
-# (see issue #24). --accept-data-loss is the same "no rollback/backup logic,
-# git revert is the recovery path" tradeoff as the rest of this script — a
-# schema-incompatible change here would need a manual look regardless.
-echo "==> syncing litellm-pgvector's database schema"
-docker compose up -d --wait pgvector-db litellm-pgvector
-docker compose exec -T litellm-pgvector prisma db push --accept-data-loss
-
 echo "==> status"
 docker compose ps
@@ -1,4 +0,0 @@
-.env
-__pycache__/*
-venv/*
-venv
@@ -1,32 +0,0 @@
-FROM python:3.11-slim
-
-# Set environment variables
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONUNBUFFERED=1
-ENV PYTHONPATH=/app
-
-# Install system dependencies
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    curl \
-    postgresql-client \
-    && rm -rf /var/lib/apt/lists/*
-
-# Set work directory
-WORKDIR /app
-
-# Install Python dependencies
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Copy project
-COPY . .
-
-# Generate Prisma client
-RUN prisma generate
-
-# Expose port
-EXPOSE 8000
-
-# Command to run the application
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] 
@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2025 Berri AI
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
@@ -1,388 +0,0 @@
-# OpenAI Vector Stores API with PGVector
-
-A FastAPI application that provides OpenAI-compatible vector store endpoints using PGVector and LiteLLM proxy for embeddings.
-
-## Features
-
- 🔌 OpenAI-compatible API endpoints
- 🗄️ PGVector for efficient vector storage and similarity search
- 🎛️ Configurable database field mappings
- 🔄 LiteLLM proxy integration for any embedding model
- 🐳 Docker support
- ⚡ FastAPI with async support
-
-## API Endpoints
-
-### 1. Create Vector Store
-```bash
-curl -X POST \
-  http://localhost:8000/v1/vector_stores \
-  -H "Authorization: Bearer your-api-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "name": "Support FAQ"
-  }'
-```
-
-### 2. List Vector Stores
-```bash
-# List all vector stores
-curl -X GET \
-  http://localhost:8000/v1/vector_stores \
-  -H "Authorization: Bearer your-api-key"
-
-# List with pagination (limit and after parameters)
-curl -X GET \
-  "http://localhost:8000/v1/vector_stores?limit=10&after=vs_abc123" \
-  -H "Authorization: Bearer your-api-key"
-```
-
-### 3. Add Single Embedding to Vector Store
-```bash
-curl -X POST \
-  http://localhost:8000/v1/vector_stores/vs_abc123/embeddings \
-  -H "Authorization: Bearer your-api-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "content": "Our return policy allows returns within 30 days of purchase.",
-    "embedding": [0.1, 0.2, 0.3, ...],
-    "metadata": {
-      "category": "returns",
-      "source": "faq",
-      "id": "return_policy_1"
-    }
-  }'
-```
-
-### 4. Add Multiple Embeddings (Batch)
-```bash
-curl -X POST \
-  http://localhost:8000/v1/vector_stores/vs_abc123/embeddings/batch \
-  -H "Authorization: Bearer your-api-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "embeddings": [
-      {
-        "content": "Our return policy allows returns within 30 days of purchase.",
-        "embedding": [0.1, 0.2, 0.3, ...],
-        "metadata": {"category": "returns"}
-      },
-      {
-        "content": "Shipping is free for orders over $50.",
-        "embedding": [0.4, 0.5, 0.6, ...],
-        "metadata": {"category": "shipping"}
-      }
-    ]
-  }'
-```
-
-### 5. Search Vector Store
-```bash
-curl -X POST \
-  http://localhost:8000/v1/vector_stores/vs_abc123/search \
-  -H "Authorization: Bearer your-api-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "query": "What is the return policy?",
-    "limit": 20,
-    "filters": {"category": "support"}
-  }'
-```
-
-## Configuration
-
-### Environment Variables
-
-Create a `.env` file with the following configuration:
-
-```bash
-# Database Configuration
-DATABASE_URL="postgresql://username:password@localhost:5432/vectordb?schema=public"
-
-# API Configuration
-SERVER_API_KEY="your-api-key-here"
-
-# Server Configuration
-HOST="0.0.0.0"
-PORT=8000
-
-# LiteLLM Proxy Configuration
-EMBEDDING__MODEL="text-embedding-ada-002"
-EMBEDDING__BASE_URL="http://localhost:4000"
-EMBEDDING__API_KEY="sk-1234"
-EMBEDDING__DIMENSIONS=1536
-
-# Database Field Configuration (optional)
-DB_FIELDS__ID_FIELD="id"
-DB_FIELDS__CONTENT_FIELD="content"
-DB_FIELDS__METADATA_FIELD="metadata"
-DB_FIELDS__EMBEDDING_FIELD="embedding"
-DB_FIELDS__VECTOR_STORE_ID_FIELD="vector_store_id"
-DB_FIELDS__CREATED_AT_FIELD="created_at"
-```
-
-### Database Field Mapping
-
-You can customize the database field names by setting environment variables:
-
- `DB_FIELDS__ID_FIELD` - Primary key field (default: "id")
- `DB_FIELDS__CONTENT_FIELD` - Text content field (default: "content")
- `DB_FIELDS__METADATA_FIELD` - JSON metadata field (default: "metadata")
- `DB_FIELDS__EMBEDDING_FIELD` - Vector embedding field (default: "embedding")
- `DB_FIELDS__VECTOR_STORE_ID_FIELD` - Foreign key field (default: "vector_store_id")
- `DB_FIELDS__CREATED_AT_FIELD` - Timestamp field (default: "created_at")
-
-### LiteLLM Proxy Configuration
-
-The application uses LiteLLM proxy for embeddings. Configure it with:
-
- `EMBEDDING__MODEL` - Model name (e.g., "text-embedding-ada-002")
- `EMBEDDING__BASE_URL` - LiteLLM proxy URL (e.g., "http://localhost:4000")
- `EMBEDDING__API_KEY` - LiteLLM proxy API key
- `EMBEDDING__DIMENSIONS` - Embedding dimensions (default: 1536)
-
-## Setup and Installation
-
-### 1. Install Dependencies
-
-```bash
-pip install -r requirements.txt
-```
-
-### 2. Database Setup
-
-```bash
-# Generate Prisma client
-prisma generate
-
-# Run database migrations
-prisma db push
-```
-
-### 3. Set up LiteLLM Proxy
-
-Start LiteLLM proxy pointing to your preferred embedding model:
-
-```bash
-# Example: Start LiteLLM proxy for OpenAI
-litellm --model text-embedding-ada-002 --port 4000
-```
-
-### 4. Run the Application
-
-```bash
-python main.py
-```
-
-Or using uvicorn directly:
-
-```bash
-uvicorn main:app --host 0.0.0.0 --port 8000 --reload
-```
-
-## Docker Deployment
-
-### Build and run with Docker:
-
-```bash
-# Build the image
-docker build -t vector-store-api .
-
-# Run the container
-docker run -p 8000:8000 --env-file .env vector-store-api
-```
-
-## Database Schema
-
-The application uses two main tables:
-
-### vector_stores
- `id` (string, primary key)
- `name` (string)
- `file_counts` (json)
- `status` (string)
- `usage_bytes` (integer)
- `created_at` (timestamp)
- `expires_after` (json, optional)
- `expires_at` (timestamp, optional)
- `last_active_at` (timestamp, optional)
- `metadata` (json, optional)
-
-### embeddings
- `id` (string, primary key)
- `vector_store_id` (string, foreign key)
- `content` (string)
- `embedding` (vector(1536))
- `metadata` (json, optional)
- `created_at` (timestamp)
-
-## Supported Models
-
-Any embedding model supported by LiteLLM proxy can be used. Examples:
-
- OpenAI: `text-embedding-ada-002`, `text-embedding-3-small`, `text-embedding-3-large`
- Cohere: `embed-english-v3.0`, `embed-multilingual-v3.0`
- Voyage: `voyage-2`, `voyage-large-2`
- And many more...
-
-## API Response Format
-
-### Vector Store Response
-```json
-{
-  "id": "vs_abc123",
-  "object": "vector_store",
-  "created_at": 1699024800,
-  "name": "Support FAQ",
-  "usage_bytes": 0,
-  "file_counts": {
-    "in_progress": 0,
-    "completed": 0,
-    "failed": 0,
-    "cancelled": 0,
-    "total": 0
-  },
-  "status": "completed",
-  "metadata": {}
-}
-```
-
-### Vector Store List Response
-```json
-{
-  "object": "list",
-  "data": [
-    {
-      "id": "vs_abc123",
-      "object": "vector_store",
-      "created_at": 1699024800,
-      "name": "Support FAQ",
-      "usage_bytes": 1024,
-      "file_counts": {"completed": 5, "total": 5},
-      "status": "completed",
-      "metadata": {}
-    }
-  ],
-  "first_id": "vs_abc123",
-  "last_id": "vs_def456",
-  "has_more": false
-}
-```
-
-### Search Response
-```json
-{
-  "object": "vector_store.search",
-  "data": [
-    {
-      "id": "emb_123",
-      "content": "Return policy text...",
-      "score": 0.95,
-      "metadata": {"category": "support"}
-    }
-  ],
-  "usage": {
-    "total_tokens": 1
-  }
-}
-```
-
-## Example Search Request
-
-```bash
-curl -X POST \
-  http://localhost:8000/v1/vector_stores/vs_support_faq/search \
-  -H "Authorization: Bearer sk-1234" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "query": "How do I return an item?",
-    "limit": 5,
-    "return_metadata": true
-  }'
-```
-
-## Health Check
-
-```bash
-curl http://localhost:8000/health
-```
-
-## Migrating Existing Data
-
-If you have an existing database with embeddings and content, you can easily migrate using the embedding APIs:
-
-### 1. Create Vector Store
-First, create a vector store for your data:
-
-```bash
-curl -X POST \
-  http://localhost:8000/v1/vector_stores \
-  -H "Authorization: Bearer your-api-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "name": "Migrated Data",
-    "metadata": {"source": "legacy_system"}
-  }'
-```
-
-### 2. Batch Insert Embeddings
-Use the batch endpoint to efficiently insert multiple embeddings:
-
-```bash
-curl -X POST \
-  http://localhost:8000/v1/vector_stores/vs_your_id/embeddings/batch \
-  -H "Authorization: Bearer your-api-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "embeddings": [
-      {
-        "content": "Your text content here",
-        "embedding": [0.1, 0.2, 0.3, ...1536 dimensions...],
-        "metadata": {"source_id": "doc_123", "category": "support"}
-      }
-    ]
-  }'
-```
-
-### 3. Migration Script Example
-
-Here's a Python script example for migrating from an existing database:
-
-```python
-import psycopg2
-import requests
-import json
-
-# Connect to your existing database
-conn = psycopg2.connect("your_existing_db_url")
-cur = conn.cursor()
-
-# Fetch existing data
-cur.execute("SELECT content, embedding, metadata FROM your_table")
-rows = cur.fetchall()
-
-# Prepare batch data
-embeddings = []
-for content, embedding, metadata in rows:
-    embeddings.append({
-        "content": content,
-        "embedding": embedding.tolist(),  # Convert numpy array to list
-        "metadata": metadata or {}
-    })
-
-# Send batch to API
-response = requests.post(
-    "http://localhost:8000/v1/vector_stores/your_vector_store_id/embeddings/batch",
-    headers={
-        "Authorization": "Bearer your-api-key",
-        "Content-Type": "application/json"
-    },
-    json={"embeddings": embeddings}
-)
-
-print(f"Migrated {len(embeddings)} embeddings")
-```
-
-## License
-
-MIT License
@@ -1,25 +0,0 @@
-Vendored from https://github.com/BerriAI/litellm-pgvector at commit
-`b553f84a32f580b4303297df5567f25912b59d93` (main, 2026-09-02). See
-`docker-compose.yml`'s `litellm-pgvector` service comment for why this is
-vendored instead of built from a remote git context.
-
-**Local changes on top of that commit** (found smoke-testing issue #24 —
-without these, the store can never be searched or written to):
-
- `prisma/schema.prisma`: `Embedding.embedding` was `vector(1536)`
-  (OpenAI ada-002's size); changed to `vector(768)` to match this stack's
-  actual embedding model (nomic-embed-text-v1.5).
- `models.py` / `main.py`: `POST /v1/vector_stores` always minted a random
-  UUID for the new store's `id`, ignoring anything the caller asked for.
-  Added an optional `id` field to `VectorStoreCreateRequest` and made
-  `create_vector_store` use it when given — `litellm-config.yaml`'s
-  `vector_store_registry` addresses this store by a fixed id
-  (`memory-and-notes`), which never matched a real row otherwise.
-
-Re-applying these after a re-vendor: diff this directory against upstream
-before overwriting, or just redo the three edits above.
-
-To update: `git clone https://github.com/BerriAI/litellm-pgvector.git`
-somewhere, copy everything except `.git/` over this directory, re-apply the
-local changes above, update the commit hash above, and run
-`./scripts/update.sh`.
@@ -1,60 +0,0 @@
-from typing import Dict, Optional
-from pydantic import BaseModel
-from pydantic_settings import BaseSettings
-
-
-class DatabaseFieldConfig(BaseModel):
-    """Configuration for database field mappings"""
-    id_field: str = "id"
-    content_field: str = "content"
-    metadata_field: str = "metadata"
-    embedding_field: str = "embedding"
-    vector_store_id_field: str = "vector_store_id"
-    created_at_field: str = "created_at"
-
-
-class EmbeddingConfig(BaseModel):
-    """Configuration for embedding generation via LiteLLM proxy"""
-    model: str = "text-embedding-ada-002"
-    base_url: str = "http://localhost:4000"  # LiteLLM proxy URL
-    api_key: str = "sk-1234"  # LiteLLM proxy API key
-    dimensions: int = 1536
-
-
-class Settings(BaseSettings):
-    """Application settings"""
-    # Database configuration
-    database_url: str = "postgresql://username:password@localhost:5432/vectordb?schema=public"
-    
-    # API configuration
-    server_api_key: str = "your-api-key-here"
-    port: int = 8000
-    host: str = "0.0.0.0"
-    
-    # Database field mappings
-    db_fields: DatabaseFieldConfig = DatabaseFieldConfig()
-    
-    # Embedding configuration
-    embedding: EmbeddingConfig = EmbeddingConfig()
-    
-    class Config:
-        env_file = ".env"
-        env_nested_delimiter = "__"
-        case_sensitive = False
-        
-        # Allow environment variables like:
-        # DB_FIELDS__ID_FIELD=custom_id
-        # EMBEDDING__MODEL=text-embedding-3-small
-        # EMBEDDING__API_BASE=https://api.openai.com/v1
-        
-    @property
-    def table_names(self) -> Dict[str, str]:
-        """Get table names"""
-        return {
-            "vector_stores": "vector_stores",
-            "embeddings": "embeddings"
-        }
-
-
-# Global settings instance
-settings = Settings() 
@@ -1,90 +0,0 @@
-from typing import List, Optional
-from config import settings, EmbeddingConfig
-from litellm.types.utils import EmbeddingResponse
-import litellm
-import logging
-
-class EmbeddingService:
-    """Service for generating embeddings using OpenAI SDK pointed at LiteLLM proxy"""
-    
-    def __init__(self, config: Optional[EmbeddingConfig] = None):
-        self.config = config or settings.embedding
-
-
-    
-    async def generate_embedding(self, text: str) -> List[float]:
-        """
-        Generate embedding for a single text using LiteLLM proxy
-        
-        Args:
-            text: Text to embed
-            
-        Returns:
-            List of floats representing the embedding vector
-        """
-        try:
-            response: EmbeddingResponse = await litellm.aembedding(
-                model=self.config.model,
-                input=[text],
-                api_base=self.config.base_url,
-                api_key=self.config.api_key
-            )
-            logging.debug(f"Embedding response: {response}")
-            
-            # Extract embedding from response
-            embedding = response.data[0]["embedding"]
-            
-            # Validate embedding dimensions
-            if len(embedding) != self.config.dimensions:
-                raise ValueError(
-                    f"Expected embedding dimension {self.config.dimensions}, "
-                    f"got {len(embedding)}"
-                )
-            
-            return embedding
-            
-        except Exception as e:
-            raise RuntimeError(f"Failed to generate embedding: {str(e)}")
-    
-    async def generate_embeddings(self, texts: List[str]) -> List[List[float]]:
-        """
-        Generate embeddings for multiple texts
-        
-        Args:
-            texts: List of texts to embed
-            
-        Returns:
-            List of embedding vectors
-        """
-        try:
-            # Generate embeddings using LiteLLM
-            response = await litellm.aembedding(
-                model=self.config.model,
-                input=texts,
-                api_base=self.config.base_url,
-                api_key=self.config.api_key
-            )
-            
-            # Extract embeddings from response
-            embeddings = [item.embedding for item in response.data]
-            
-            # Validate embedding dimensions
-            for i, embedding in enumerate(embeddings):
-                if len(embedding) != self.config.dimensions:
-                    raise ValueError(
-                        f"Expected embedding dimension {self.config.dimensions} for text {i}, "
-                        f"got {len(embedding)}"
-                    )
-            
-            return embeddings
-            
-        except Exception as e:
-            raise RuntimeError(f"Failed to generate embeddings: {str(e)}")
-    
-    def update_config(self, new_config: EmbeddingConfig):
-        """Update the embedding configuration"""
-        self.config = new_config
-
-
-# Global embedding service instance
-embedding_service = EmbeddingService() 
@@ -1,530 +0,0 @@
-import os
-import asyncio
-import time
-from typing import List, Optional
-from fastapi import FastAPI, HTTPException, Depends, Header
-from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
-from fastapi.middleware.cors import CORSMiddleware
-from prisma import Prisma
-from dotenv import load_dotenv
-
-from models import (
-    VectorStoreCreateRequest,
-    VectorStoreResponse,
-    VectorStoreSearchRequest,
-    VectorStoreSearchResponse,
-    SearchResult,
-    EmbeddingCreateRequest,
-    EmbeddingResponse,
-    EmbeddingBatchCreateRequest,
-    EmbeddingBatchCreateResponse,
-    VectorStoreListResponse,
-    ContentChunk
-)
-from config import settings
-from embedding_service import embedding_service
-
-load_dotenv()
-
-app = FastAPI(
-    title="OpenAI Vector Stores API",
-    description="OpenAI-compatible Vector Stores API using PGVector",
-    version="1.0.0"
-)
-
-# CORS middleware
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=["*"],
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-# Global Prisma client
-db = Prisma()
-
-security = HTTPBearer()
-
-
-async def get_api_key(credentials: HTTPAuthorizationCredentials = Depends(security)):
-    """Validate API key from Authorization header"""
-    expected_key = settings.server_api_key
-    if credentials.credentials != expected_key:
-        raise HTTPException(status_code=401, detail="Invalid API key")
-    return credentials.credentials
-
-
-@app.on_event("startup")
-async def startup():
-    """Connect to database on startup"""
-    await db.connect()
-
-
-@app.on_event("shutdown")
-async def shutdown():
-    """Disconnect from database on shutdown"""
-    await db.disconnect()
-
-
-async def generate_query_embedding(query: str) -> List[float]:
-    """
-    Generate an embedding for the query using LiteLLM
-    """
-    return await embedding_service.generate_embedding(query)
-
-
-@app.post("/v1/vector_stores", response_model=VectorStoreResponse)
-async def create_vector_store(
-    request: VectorStoreCreateRequest,
-    api_key: str = Depends(get_api_key)
-):
-    """
-    Create a new vector store.
-    """
-    try:
-        # Use raw SQL to insert the vector store with configurable table/field names
-        vector_store_table = settings.table_names["vector_stores"]
-
-        # ponytail: honor a caller-supplied id (request.id) instead of
-        # always minting one — litellm's vector_store_registry addresses
-        # this store by a fixed id (see litellm-config.yaml), which never
-        # matched anything when this always generated a random UUID.
-        import uuid as _uuid
-        vector_store_id = request.id or str(_uuid.uuid4())
-
-        result = await db.query_raw(
-            f"""
-            INSERT INTO {vector_store_table} (id, name, file_counts, status, usage_bytes, expires_after, metadata, created_at)
-            VALUES ($1, $2, $3, $4, $5, $6, $7, NOW())
-            RETURNING id, name, file_counts, status, usage_bytes, expires_after, expires_at, last_active_at, metadata,
-                     EXTRACT(EPOCH FROM created_at)::bigint as created_at_timestamp
-            """,
-            vector_store_id,
-            request.name,
-            {"in_progress": 0, "completed": 0, "failed": 0, "cancelled": 0, "total": 0},
-            "completed",
-            0,
-            request.expires_after,
-            request.metadata or {}
-        )
-        
-        if not result:
-            raise HTTPException(status_code=500, detail="Failed to create vector store")
-            
-        vector_store = result[0]
-        
-        # Convert to response format
-        created_at = int(vector_store["created_at_timestamp"])
-        expires_at = int(vector_store["expires_at"].timestamp()) if vector_store.get("expires_at") else None
-        last_active_at = int(vector_store["last_active_at"].timestamp()) if vector_store.get("last_active_at") else None
-        
-        return VectorStoreResponse(
-            id=vector_store["id"],
-            created_at=created_at,
-            name=vector_store["name"],
-            usage_bytes=vector_store["usage_bytes"] or 0,
-            file_counts=vector_store["file_counts"] or {"in_progress": 0, "completed": 0, "failed": 0, "cancelled": 0, "total": 0},
-            status=vector_store["status"],
-            expires_after=vector_store["expires_after"],
-            expires_at=expires_at,
-            last_active_at=last_active_at,
-            metadata=vector_store["metadata"]
-        )
-        
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=f"Failed to create vector store: {str(e)}")
-
-
-@app.get("/v1/vector_stores", response_model=VectorStoreListResponse)
-async def list_vector_stores(
-    limit: Optional[int] = 20,
-    after: Optional[str] = None,
-    before: Optional[str] = None,
-    api_key: str = Depends(get_api_key)
-):
-    """
-    List vector stores with optional pagination.
-    """
-    try:
-        limit = min(limit or 20, 100)  # Cap at 100 results
-        
-        vector_store_table = settings.table_names["vector_stores"]
-        
-        # Build base query
-        base_query = f"""
-        SELECT id, name, file_counts, status, usage_bytes, expires_after, expires_at, last_active_at, metadata,
-               EXTRACT(EPOCH FROM created_at)::bigint as created_at_timestamp
-        FROM {vector_store_table}
-        """
-        
-        # Add pagination conditions
-        conditions = []
-        params = []
-        param_count = 1
-        
-        if after:
-            conditions.append(f"id > ${param_count}")
-            params.append(after)
-            param_count += 1
-            
-        if before:
-            conditions.append(f"id < ${param_count}")
-            params.append(before)
-            param_count += 1
-        
-        if conditions:
-            base_query += " WHERE " + " AND ".join(conditions)
-        
-        # Add ordering and limit
-        final_query = base_query + f" ORDER BY created_at DESC LIMIT {limit + 1}"
-        
-        # Execute query
-        results = await db.query_raw(final_query, *params)
-        
-        # Check if there are more results
-        has_more = len(results) > limit
-        if has_more:
-            results = results[:limit]  # Remove extra result
-        
-        # Convert to response format
-        vector_stores = []
-        for row in results:
-            created_at = int(row["created_at_timestamp"])
-            expires_at = int(row["expires_at"].timestamp()) if row.get("expires_at") else None
-            last_active_at = int(row["last_active_at"].timestamp()) if row.get("last_active_at") else None
-            
-            vector_store = VectorStoreResponse(
-                id=row["id"],
-                created_at=created_at,
-                name=row["name"],
-                usage_bytes=row["usage_bytes"] or 0,
-                file_counts=row["file_counts"] or {"in_progress": 0, "completed": 0, "failed": 0, "cancelled": 0, "total": 0},
-                status=row["status"],
-                expires_after=row["expires_after"],
-                expires_at=expires_at,
-                last_active_at=last_active_at,
-                metadata=row["metadata"]
-            )
-            vector_stores.append(vector_store)
-        
-        # Determine first_id and last_id
-        first_id = vector_stores[0].id if vector_stores else None
-        last_id = vector_stores[-1].id if vector_stores else None
-        
-        return VectorStoreListResponse(
-            data=vector_stores,
-            first_id=first_id,
-            last_id=last_id,
-            has_more=has_more
-        )
-        
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        raise HTTPException(status_code=500, detail=f"Failed to list vector stores: {str(e)}")
-
-
-@app.post("/v1/vector_stores/{vector_store_id}/search", response_model=VectorStoreSearchResponse)
-@app.post("/vector_stores/{vector_store_id}/search", response_model=VectorStoreSearchResponse)
-async def search_vector_store(
-    vector_store_id: str,
-    request: VectorStoreSearchRequest,
-    api_key: str = Depends(get_api_key)
-):
-    """
-    Search a vector store for similar content.
-    """
-    try:
-        # Check if vector store exists
-        vector_store_table = settings.table_names["vector_stores"]
-        vector_store_result = await db.query_raw(
-            f"SELECT id FROM {vector_store_table} WHERE id = $1",
-            vector_store_id
-        )
-        if not vector_store_result:
-            raise HTTPException(status_code=404, detail="Vector store not found")
-        
-        # Generate embedding for query
-        query_embedding = await generate_query_embedding(request.query)
-        query_vector_str = "[" + ",".join(map(str, query_embedding)) + "]"
-        
-        # Build the raw SQL query for vector similarity search
-        limit = min(request.limit or 20, 100)  # Cap at 100 results
-        
-        # Base query with vector similarity using cosine distance
-        # Use configurable field names
-        fields = settings.db_fields
-        table_name = settings.table_names["embeddings"]
-        
-        # Build query with proper parameter placeholders for Prisma
-        param_count = 1
-        query_params = [query_vector_str, vector_store_id]
-        
-        base_query = f"""
-        SELECT 
-            {fields.id_field},
-            {fields.content_field},
-            {fields.metadata_field},
-            ({fields.embedding_field} <=> ${param_count}::vector) as distance
-        FROM {table_name} 
-        WHERE {fields.vector_store_id_field} = ${param_count + 1}
-        """
-        param_count += 2
-        
-        # Add metadata filters if provided
-        filter_conditions = []
-        
-        if request.filters:
-            for key, value in request.filters.items():
-                filter_conditions.append(f"{fields.metadata_field}->>${param_count} = ${param_count + 1}")
-                query_params.extend([key, str(value)])
-                param_count += 2
-        
-        if filter_conditions:
-            base_query += " AND " + " AND ".join(filter_conditions)
-        
-        # Add ordering and limit
-        final_query = base_query + f" ORDER BY distance ASC LIMIT {limit}"
-        
-        # Execute the query
-        results = await db.query_raw(final_query, *query_params)
-        
-        # Convert results to SearchResult objects
-        search_results = []
-        for row in results:
-            # Convert distance to similarity score (1 - normalized_distance)
-            # Cosine distance ranges from 0 (identical) to 2 (opposite)
-            similarity_score = max(0, 1 - (row['distance'] / 2))
-            
-            # Extract filename from metadata or use a default
-            metadata = row[fields.metadata_field] or {}
-            filename = metadata.get('filename', 'document.txt')
-            
-            content_chunks = [ContentChunk(type="text", text=row[fields.content_field])]
-            
-            result = SearchResult(
-                file_id=row[fields.id_field],
-                filename=filename,
-                score=similarity_score,
-                attributes=metadata if request.return_metadata else None,
-                content=content_chunks
-            )
-            search_results.append(result)
-        
-        return VectorStoreSearchResponse(
-            search_query=request.query,
-            data=search_results,
-            has_more=False,  # TODO: Implement pagination
-            next_page=None
-        )
-        
-    except HTTPException:
-        raise
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}")
-
-
-@app.post("/v1/vector_stores/{vector_store_id}/embeddings", response_model=EmbeddingResponse)
-async def create_embedding(
-    vector_store_id: str,
-    request: EmbeddingCreateRequest,
-    api_key: str = Depends(get_api_key)
-):
-    """
-    Add a single embedding to a vector store.
-    """
-    try:
-        # Check if vector store exists
-        vector_store_table = settings.table_names["vector_stores"]
-        vector_store_result = await db.query_raw(
-            f"SELECT id FROM {vector_store_table} WHERE id = $1",
-            vector_store_id
-        )
-        if not vector_store_result:
-            raise HTTPException(status_code=404, detail="Vector store not found")
-        
-        # Convert embedding to vector string format
-        embedding_vector_str = "[" + ",".join(map(str, request.embedding)) + "]"
-        
-        # Insert embedding using configurable field names
-        fields = settings.db_fields
-        table_name = settings.table_names["embeddings"]
-        
-        result = await db.query_raw(
-            f"""
-            INSERT INTO {table_name} ({fields.id_field}, {fields.vector_store_id_field}, {fields.content_field}, 
-                                     {fields.embedding_field}, {fields.metadata_field}, {fields.created_at_field})
-            VALUES (gen_random_uuid(), $1, $2, $3::vector, $4, NOW())
-            RETURNING {fields.id_field}, {fields.vector_store_id_field}, {fields.content_field}, 
-                     {fields.metadata_field}, EXTRACT(EPOCH FROM {fields.created_at_field})::bigint as created_at_timestamp
-            """,
-            vector_store_id,
-            request.content,
-            embedding_vector_str,
-            request.metadata or {}
-        )
-        
-        if not result:
-            raise HTTPException(status_code=500, detail="Failed to create embedding")
-            
-        embedding = result[0]
-        
-        # Update vector store statistics
-        await db.query_raw(
-            f"""
-            UPDATE {vector_store_table} 
-            SET 
-                file_counts = jsonb_set(
-                    jsonb_set(
-                        COALESCE(file_counts, '{{"in_progress": 0, "completed": 0, "failed": 0, "cancelled": 0, "total": 0}}'::jsonb),
-                        '{{completed}}',
-                        (COALESCE(file_counts->>'completed', '0')::int + 1)::text::jsonb
-                    ),
-                    '{{total}}',
-                    (COALESCE(file_counts->>'total', '0')::int + 1)::text::jsonb
-                ),
-                usage_bytes = COALESCE(usage_bytes, 0) + LENGTH($2),
-                last_active_at = NOW()
-            WHERE id = $1
-            """,
-            vector_store_id,
-            request.content
-        )
-        
-        return EmbeddingResponse(
-            id=embedding[fields.id_field],
-            vector_store_id=embedding[fields.vector_store_id_field],
-            content=embedding[fields.content_field],
-            metadata=embedding[fields.metadata_field],
-            created_at=int(embedding["created_at_timestamp"])
-        )
-        
-    except HTTPException:
-        raise
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        raise HTTPException(status_code=500, detail=f"Failed to create embedding: {str(e)}")
-
-
-@app.post("/v1/vector_stores/{vector_store_id}/embeddings/batch", response_model=EmbeddingBatchCreateResponse)
-async def create_embeddings_batch(
-    vector_store_id: str,
-    request: EmbeddingBatchCreateRequest,
-    api_key: str = Depends(get_api_key)
-):
-    """
-    Add multiple embeddings to a vector store in batch.
-    """
-    try:
-        # Check if vector store exists
-        vector_store_table = settings.table_names["vector_stores"]
-        vector_store_result = await db.query_raw(
-            f"SELECT id FROM {vector_store_table} WHERE id = $1",
-            vector_store_id
-        )
-        if not vector_store_result:
-            raise HTTPException(status_code=404, detail="Vector store not found")
-        
-        if not request.embeddings:
-            raise HTTPException(status_code=400, detail="No embeddings provided")
-        
-        # Prepare batch insert
-        fields = settings.db_fields
-        table_name = settings.table_names["embeddings"]
-        
-        # Build VALUES clause for batch insert
-        values_clauses = []
-        params = []
-        param_count = 1
-        
-        for embedding_req in request.embeddings:
-            embedding_vector_str = "[" + ",".join(map(str, embedding_req.embedding)) + "]"
-            values_clauses.append(f"(gen_random_uuid(), ${param_count}, ${param_count + 1}, ${param_count + 2}::vector, ${param_count + 3}, NOW())")
-            params.extend([
-                vector_store_id,
-                embedding_req.content,
-                embedding_vector_str,
-                embedding_req.metadata or {}
-            ])
-            param_count += 4
-        
-        values_clause = ", ".join(values_clauses)
-        
-        # Execute batch insert
-        result = await db.query_raw(
-            f"""
-            INSERT INTO {table_name} ({fields.id_field}, {fields.vector_store_id_field}, {fields.content_field}, 
-                                     {fields.embedding_field}, {fields.metadata_field}, {fields.created_at_field})
-            VALUES {values_clause}
-            RETURNING {fields.id_field}, {fields.vector_store_id_field}, {fields.content_field}, 
-                     {fields.metadata_field}, EXTRACT(EPOCH FROM {fields.created_at_field})::bigint as created_at_timestamp
-            """,
-            *params
-        )
-        
-        if not result:
-            raise HTTPException(status_code=500, detail="Failed to create embeddings")
-        
-        # Calculate total content length for usage bytes update
-        total_content_length = sum(len(emb.content) for emb in request.embeddings)
-        
-        # Update vector store statistics
-        await db.query_raw(
-            f"""
-            UPDATE {vector_store_table} 
-            SET 
-                file_counts = jsonb_set(
-                    jsonb_set(
-                        COALESCE(file_counts, '{{"in_progress": 0, "completed": 0, "failed": 0, "cancelled": 0, "total": 0}}'::jsonb),
-                        '{{completed}}',
-                        (COALESCE(file_counts->>'completed', '0')::int + $2)::text::jsonb
-                    ),
-                    '{{total}}',
-                    (COALESCE(file_counts->>'total', '0')::int + $2)::text::jsonb
-                ),
-                usage_bytes = COALESCE(usage_bytes, 0) + $3,
-                last_active_at = NOW()
-            WHERE id = $1
-            """,
-            vector_store_id,
-            len(request.embeddings),
-            total_content_length
-        )
-        
-        # Convert results to response format
-        embeddings = []
-        for row in result:
-            embeddings.append(EmbeddingResponse(
-                id=row[fields.id_field],
-                vector_store_id=row[fields.vector_store_id_field],
-                content=row[fields.content_field],
-                metadata=row[fields.metadata_field],
-                created_at=int(row["created_at_timestamp"])
-            ))
-        
-        return EmbeddingBatchCreateResponse(
-            data=embeddings,
-            created=int(time.time())
-        )
-        
-    except HTTPException:
-        raise
-    except Exception as e:
-        import traceback
-        traceback.print_exc()
-        raise HTTPException(status_code=500, detail=f"Failed to create embeddings batch: {str(e)}")
-
-
-@app.get("/health")
-async def health_check():
-    """Health check endpoint"""
-    return {"status": "healthy", "timestamp": int(time.time())}
-
-
-if __name__ == "__main__":
-    import uvicorn
-    uvicorn.run("main:app", host=settings.host, port=settings.port, reload=True) 
@@ -1,92 +0,0 @@
-from typing import Optional, Dict, Any, List
-from pydantic import BaseModel
-from datetime import datetime
-
-
-class VectorStoreCreateRequest(BaseModel):
-    name: str
-    # ponytail: not part of upstream litellm-pgvector — added locally so
-    # callers (litellm's vector_store_registry, scripts/ingest-memory.sh)
-    # can pin a human-readable id instead of getting a random UUID back.
-    # litellm-config.yaml's vector_store_registry addresses stores by a
-    # fixed vector_store_id, which only works if creation can honor it.
-    id: Optional[str] = None
-    file_ids: Optional[List[str]] = None
-    expires_after: Optional[Dict[str, Any]] = None
-    chunking_strategy: Optional[Dict[str, Any]] = None
-    metadata: Optional[Dict[str, Any]] = None
-
-
-class VectorStoreResponse(BaseModel):
-    id: str
-    object: str = "vector_store"
-    created_at: int
-    name: str
-    usage_bytes: int
-    file_counts: Dict[str, int]
-    status: str
-    expires_after: Optional[Dict[str, Any]] = None
-    expires_at: Optional[int] = None
-    last_active_at: Optional[int] = None
-    metadata: Optional[Dict[str, Any]] = None
-
-
-class VectorStoreSearchRequest(BaseModel):
-    query: str
-    limit: Optional[int] = 20
-    filters: Optional[Dict[str, Any]] = None
-    return_metadata: Optional[bool] = True
-
-
-class ContentChunk(BaseModel):
-    type: str = "text"
-    text: str
-
-
-class SearchResult(BaseModel):
-    file_id: str
-    filename: str
-    score: float
-    attributes: Optional[Dict[str, Any]] = None
-    content: List[ContentChunk]
-
-
-class VectorStoreSearchResponse(BaseModel):
-    object: str = "vector_store.search_results.page"
-    search_query: str
-    data: List[SearchResult]
-    has_more: bool = False
-    next_page: Optional[str] = None
-
-
-class EmbeddingCreateRequest(BaseModel):
-    content: str
-    embedding: List[float]
-    metadata: Optional[Dict[str, Any]] = None
-
-
-class EmbeddingResponse(BaseModel):
-    id: str
-    object: str = "embedding"
-    vector_store_id: str
-    content: str
-    metadata: Optional[Dict[str, Any]] = None
-    created_at: int
-
-
-class EmbeddingBatchCreateRequest(BaseModel):
-    embeddings: List[EmbeddingCreateRequest]
-
-
-class EmbeddingBatchCreateResponse(BaseModel):
-    object: str = "embedding.batch"
-    data: List[EmbeddingResponse]
-    created: int
-
-
-class VectorStoreListResponse(BaseModel):
-    object: str = "list"
-    data: List[VectorStoreResponse]
-    first_id: Optional[str] = None
-    last_id: Optional[str] = None
-    has_more: bool = False
@@ -1,45 +0,0 @@
-// This is your Prisma schema file,
-// learn more about it in the docs: https://pris.ly/d/prisma-schema
-
-generator client {
-  provider = "prisma-client-py"
-}
-
-datasource db {
-  provider = "postgresql"
-  url      = env("DATABASE_URL")
-}
-
-model VectorStore {
-  id           String   @id @default(cuid())
-  name         String
-  file_counts  Json?
-  status       String   @default("completed")
-  usage_bytes  Int?     @default(0)
-  created_at   DateTime @default(now())
-  expires_after Json?
-  expires_at   DateTime?
-  last_active_at DateTime?
-  metadata     Json?
-  embeddings   Embedding[]
-  
-  @@map("vector_stores")
-}
-
-model Embedding {
-  id              String      @id @default(cuid())
-  vector_store_id String
-  content         String
-  // 768, not OpenAI's ada-002-sized 1536 — this stack's embedding_model is
-  // nomic-embed-text-v1.5 (see docker-compose.yml's embedding-server and
-  // litellm-config.yaml's local-embedding entry), confirmed 768-dim live
-  // against /v1/embeddings. A push with the wrong size here makes every
-  // insert fail on a pgvector dimension mismatch.
-  embedding       Unsupported("vector(768)")
-  metadata        Json?
-  created_at      DateTime    @default(now())
-  
-  vector_store    VectorStore @relation(fields: [vector_store_id], references: [id], onDelete: Cascade)
-  
-  @@map("embeddings")
-} 
@@ -1,10 +0,0 @@
-fastapi==0.104.1
-uvicorn[standard]==0.24.0
-prisma==0.11.0
-python-dotenv==1.0.0
-pydantic>=2.5.0
-psycopg2-binary==2.9.7
-pgvector==0.2.4
-python-multipart==0.0.6
-litellm==1.74.3
-pydantic-settings==2.1.0