From 52a92f650896c66a51db05bc654ca92b309c1a72 Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Sun, 6 Sep 2026 22:06:59 +0200 Subject: [PATCH] fix: llama-server-fast context-size exhaustion breaking Auto Mode classifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real failure: "Auto Mode couldn't classify this action (Classifier stage 1 unavailable)". Reproduced directly against the server: {"error":{"message":"[400]: request (6186 tokens) exceeds the available context size (4096 tokens)"... LLAMA_FAST_CTX_SIZE=8192 is the TOTAL across every LLAMA_FAST_PARALLEL slot, not per-request — the main model's own .env.example comment already calls this out, missed it when llama-server-fast was set up (#44). With PARALLEL=2 that's 4096/slot, too small for a real classifier call (hints + environment + recent tool-call history). Fixed by dropping to a single slot (LLAMA_FAST_PARALLEL=1) rather than raising ctx-size — this service doesn't need concurrent classifier calls the way the main model needs concurrent chat sessions, so this costs no extra VRAM. The full 8192 now goes to the one slot. docker compose config -q validated. Refs #5 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01MrnMEdzeQzqZE5soVEXPCx --- .env.example | 19 +++++++++++++++---- docker-compose.yml | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 619dbc4..5108412 100644 --- a/.env.example +++ b/.env.example @@ -116,11 +116,22 @@ LLAMA_FAST_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf # Same reasoning as LLAMA_GPU_LAYERS above — full GPU offload, this model # is dense too. LLAMA_FAST_GPU_LAYERS=999 -# Classifier transcripts are truncated/bounded by qwen-code itself (see its -# own Auto Mode docs) — no need for anywhere near the 27B's huge context. -# 8192 keeps this instance's KV cache negligible. +# --ctx-size is the TOTAL across every LLAMA_FAST_PARALLEL slot, not per +# request — same halving already called out for the main model above. +# Was PARALLEL=2, silently halving this to 4096/slot — too small: a real +# classifier call (hints + environment + recent tool-call history) hit +# "exceeds the available context size (4096 tokens)" in practice, which +# qwen-code surfaces as "Auto Mode couldn't classify this action +# (Classifier stage 1 unavailable)" — see issue #5. Fixed by dropping to +# a single slot instead of raising ctx-size (no extra VRAM, and this +# service doesn't need concurrent classifier calls the way the main +# model needs concurrent chat sessions) — the full 8192 now goes to the +# one slot. If hints.allow/softDeny/hardDeny ever approach their +# 50-entries-each ceiling, raise LLAMA_FAST_CTX_SIZE instead — qwen-code +# caps those at 200 chars x 150 entries plus 40,000 chars of +# historical-action context, which can exceed 8192 tokens worst-case. LLAMA_FAST_CTX_SIZE=8192 -LLAMA_FAST_PARALLEL=2 +LLAMA_FAST_PARALLEL=1 # --- ComfyUI diffusion model (Qwen-Image, FP8 — see docs/research/ # image-generation-model-choice.md and issue #42) --- diff --git a/docker-compose.yml b/docker-compose.yml index 4e18455..5c0ed1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,7 +70,7 @@ services: --port 8080 --n-gpu-layers ${LLAMA_FAST_GPU_LAYERS:-999} --ctx-size ${LLAMA_FAST_CTX_SIZE:-8192} - --parallel ${LLAMA_FAST_PARALLEL:-2} + --parallel ${LLAMA_FAST_PARALLEL:-1} --flash-attn on --cache-type-k q8_0 --cache-type-v q8_0