diff --git a/.env.example b/.env.example index 9eb87fc..3233082 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,15 @@ LLAMA_GPU_LAYERS=999 # to spare) or drop back to 131072 (128K), the last known-good value # (~25.6GB total, ~6GB headroom) — see docs/research/qwen3.8-27b-quant.md. LLAMA_CTX_SIZE=262144 +# Concurrent request slots. Was implicitly 4 (llama.cpp's compiled-in +# default) with no flag set — under concurrent subagent fan-out, 4 requests +# split the same GPU compute, so a large-context prefill can queue behind +# others long enough to blow past OmniRoute's stream-idle timeout, which then +# cancels the request (see issue-tracker notes on the timeout/cancel loop). +# Dropped to 2 so each slot gets more compute and finishes prefill sooner; +# raise back toward 4 if throughput (not latency) becomes the bottleneck +# instead. +LLAMA_PARALLEL=2 # --- Lazytainer --- # Seconds of inactivity before llama-server is stopped. 900 = 15 min. diff --git a/docker-compose.yml b/docker-compose.yml index d353aab..59ce820 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ services: --port 8080 --n-gpu-layers ${LLAMA_GPU_LAYERS:-999} --ctx-size ${LLAMA_CTX_SIZE:-131072} + --parallel ${LLAMA_PARALLEL:-2} --jinja # No published host port: llama-server is reached only via the omniroute # gateway on the ai-stack docker network now — see issue #15. Its