diff --git a/.env.example b/.env.example index 619dbc4..5108412 100644 --- a/.env.example +++ b/.env.example @@ -116,11 +116,22 @@ LLAMA_FAST_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf # Same reasoning as LLAMA_GPU_LAYERS above — full GPU offload, this model # is dense too. LLAMA_FAST_GPU_LAYERS=999 -# Classifier transcripts are truncated/bounded by qwen-code itself (see its -# own Auto Mode docs) — no need for anywhere near the 27B's huge context. -# 8192 keeps this instance's KV cache negligible. +# --ctx-size is the TOTAL across every LLAMA_FAST_PARALLEL slot, not per +# request — same halving already called out for the main model above. +# Was PARALLEL=2, silently halving this to 4096/slot — too small: a real +# classifier call (hints + environment + recent tool-call history) hit +# "exceeds the available context size (4096 tokens)" in practice, which +# qwen-code surfaces as "Auto Mode couldn't classify this action +# (Classifier stage 1 unavailable)" — see issue #5. Fixed by dropping to +# a single slot instead of raising ctx-size (no extra VRAM, and this +# service doesn't need concurrent classifier calls the way the main +# model needs concurrent chat sessions) — the full 8192 now goes to the +# one slot. If hints.allow/softDeny/hardDeny ever approach their +# 50-entries-each ceiling, raise LLAMA_FAST_CTX_SIZE instead — qwen-code +# caps those at 200 chars x 150 entries plus 40,000 chars of +# historical-action context, which can exceed 8192 tokens worst-case. LLAMA_FAST_CTX_SIZE=8192 -LLAMA_FAST_PARALLEL=2 +LLAMA_FAST_PARALLEL=1 # --- ComfyUI diffusion model (Qwen-Image, FP8 — see docs/research/ # image-generation-model-choice.md and issue #42) --- diff --git a/docker-compose.yml b/docker-compose.yml index 4e18455..5c0ed1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,7 +70,7 @@ services: --port 8080 --n-gpu-layers ${LLAMA_FAST_GPU_LAYERS:-999} --ctx-size ${LLAMA_FAST_CTX_SIZE:-8192} - --parallel ${LLAMA_FAST_PARALLEL:-2} + --parallel ${LLAMA_FAST_PARALLEL:-1} --flash-attn on --cache-type-k q8_0 --cache-type-v q8_0