From 828bd4c046848400e9038f0d62b888582dfae3bc Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Wed, 9 Sep 2026 16:26:51 +0200 Subject: [PATCH] feat(llm): dedicate a CPU-only backend for the qwen-code tool-call classifier fastModel in ~/.qwen/settings.json (permissions.autoMode.classifier) was aliased onto llama-server's own 27B connection, so every tool-call safety check queued behind whatever heavy generation was already running on that model's 2 GPU slots. Add qwen-classifier: a separate llama.cpp instance, CPU-only, running Qwen3-4B-Instruct-2507 (the smallest Qwen3 with native >=131072 context, qwen-code's requirement, without lossy RoPE scaling). Structurally isolated from llama-server's queue instead of sharing it. Sized for gameserver's ~17GiB free system RAM: q8_0/q8_0 KV at full 131072 ctx (~9.8GiB) + Q4_K_M- class weights (~2.3GiB) fits comfortably, with better KV quality than the q4_0 that would've been needed to fit this on the GPU's ~6GiB free VRAM. Co-Authored-By: Claude Sonnet 5 --- .env.example | 29 +++++++++++++++++---------- docker-compose.yml | 49 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/update.sh | 1 + 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index f7e51d9..6601878 100644 --- a/.env.example +++ b/.env.example @@ -29,18 +29,27 @@ LLAMA_GPU_LAYERS=999 # this size) — total ~25.6GB, ~6GB headroom, the same footprint the old # 131072 fp16 setting used. See docs/research/qwen3.8-27b-quant.md. LLAMA_CTX_SIZE=262144 -# Concurrent request slots. Was implicitly 4 (llama.cpp's compiled-in -# default) with no flag set — under concurrent subagent fan-out, 4 requests -# split the same GPU compute, so a large-context prefill can queue behind -# others long enough to blow past OmniRoute's stream-idle timeout, which then -# cancels the request (see issue-tracker notes on the timeout/cancel loop). -# Dropped to 2 so each slot gets more compute and finishes prefill sooner; -# raise back toward 4 if throughput (not latency) becomes the bottleneck -# instead. Each slot gets LLAMA_CTX_SIZE / LLAMA_PARALLEL tokens of context — -# real sessions have hit ~66K tokens, so don't drop LLAMA_CTX_SIZE without -# checking that per-slot number stays comfortably above observed usage. +# Concurrent request slots — the real hardware ceiling for this GPU, not a +# tunable to raise for throughput (was implicitly 4, llama.cpp's compiled-in +# default; dropped to 2 because more contended prefill was blowing requests +# past OmniRoute's idle timeout — see OMNIROUTE_STREAM_IDLE_TIMEOUT_MS below). +# The 3rd+ request now queues on llama.cpp itself instead — its own queue has +# no timeout (tools/server/server-queue.cpp), it just waits for a slot — so +# the timeout that matters moved to OmniRoute's per-connection +# providerSpecificData.timeoutMs (dashboard/API only, not in this file; see +# handoff notes in the issue tracker). Each slot gets LLAMA_CTX_SIZE / +# LLAMA_PARALLEL tokens of context — real sessions have hit ~66K tokens, so +# don't drop LLAMA_CTX_SIZE without checking that per-slot number stays +# comfortably above observed usage. LLAMA_PARALLEL=2 +# Dedicated CPU-only backend for qwen-code's tool-call harmfulness classifier +# (fastModel in ~/.qwen/settings.json) — see docker-compose.yml's +# qwen-classifier service comment for the why. 131072 ctx / q8_0 KV / Q4_K_M- +# class weights ≈ 12GiB, fits gameserver's ~17GiB free system RAM with room +# to spare. +LLAMA_CLASSIFIER_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf + # --- Lazytainer --- # Seconds of inactivity before llama-server is stopped. 900 = 15 min. LAZYTAINER_INACTIVE_TIMEOUT=900 diff --git a/docker-compose.yml b/docker-compose.yml index 0fcc7e1..c09ecee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,6 +46,39 @@ services: - "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}" - "lazytainer.group.llamaserver.minPacketThreshold=2" + # Dedicated backend for qwen-code's tool-call harmfulness classifier + # (fastModel in ~/.qwen/settings.json). Was aliased onto llama-server's own + # 27B connection — every classification call then queued behind whatever + # heavy generation was already running on that model's 2 GPU slots (issue + # tracker: OmniRoute semaphore/pr-agent investigation). CPU-only, own + # process, own queue: structurally can't contend with llama-server for a + # GPU slot. Needs >=131072 ctx (qwen-code requirement); Qwen3-4B-Instruct-2507 + # is the smallest Qwen3 that supports that natively (262144) without + # RoPE-scaling — the smaller 0.6B/1.7B/4B (non-2507) models only go to + # 40960. Sized for gameserver's ~17GiB free RAM: q8_0/q8_0 KV at full + # 131072 ctx is ~9.8GiB + ~2.3GiB Q4_K_M-class weights ≈ 12GiB, comfortable + # headroom, and better KV quality than the q4_0 that would've been needed + # to squeeze this onto the GPU's ~6GiB free VRAM instead. + qwen-classifier: + image: ghcr.io/ggml-org/llama.cpp:server + container_name: qwen-classifier + volumes: + - models:/models + command: > + -m /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf} + --host 0.0.0.0 + --port 8080 + --n-gpu-layers 0 + --ctx-size 131072 + --parallel 1 + --cache-type-k q8_0 + --cache-type-v q8_0 + --jinja + expose: + - "8080" + restart: unless-stopped + networks: [ai-stack] + # ponytail: one-off downloader, not a standing service — run via # `docker compose --profile tools run --rm downloader`. Folded into # scripts/update.sh, which runs this every time; the `test -f` guard is @@ -67,6 +100,22 @@ services: curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} + # Same pattern as downloader above, separate service so this one small + # file doesn't get re-checked/re-pulled by the big model's job. + downloader-classifier: + image: curlimages/curl:latest + profiles: ["tools"] + user: root + volumes: + - models:/models + entrypoint: ["sh", "-c"] + command: + - > + test -f /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf} && + echo "already downloaded, skipping" || + curl -L --fail --create-dirs -o /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf} + https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf} + # Fetches the three Qwen-Image FP8 files ComfyUI needs (diffusion model, # text encoder, VAE) — same test -f guard pattern as downloader above. # See docs/research/image-generation-model-choice.md and issue #42. diff --git a/scripts/update.sh b/scripts/update.sh index e1a6bae..91b0b4d 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -233,6 +233,7 @@ docker compose build --pull echo "==> ensuring models are downloaded (skips already-present files)" docker compose --profile tools run --rm downloader docker compose --profile tools run --rm downloader-fast +docker compose --profile tools run --rm downloader-classifier docker compose --profile tools run --rm downloader-comfyui echo "==> bringing up omniroute"