diff --git a/.env.example b/.env.example index 3d6e884..71a2be8 100644 --- a/.env.example +++ b/.env.example @@ -23,14 +23,11 @@ LLAMA_MODEL_FILE=Qwen3.8-27B-UD-Q4_K_XL.gguf LLAMA_GPU_LAYERS=999 # 262144 = this model's true max (max_position_embeddings in Qwen/Qwen3.8-27B's # config.json) — the largest --ctx-size llama.cpp will even accept for it. -# fp16 KV cache at full context is ~16GB (262,144 x 64KiB/token), on top of -# 17.6GB weights = ~33.6GB, which does NOT fit the 32GB R9700 — expect -# llama-server to fail to allocate and refuse to start at this setting. -# Chosen anyway per explicit request to run at the model's real ceiling -# rather than a VRAM-safe value. To actually get a working max: quantize the -# KV cache (--cache-type-k/v q8_0, roughly halves it to ~8GB, fits with room -# to spare) or drop back to 131072 (128K), the last known-good value -# (~25.6GB total, ~6GB headroom) — see docs/research/qwen3.8-27b-quant.md. +# fp16 KV cache at full context would be ~16GB, on top of 17.6GB weights = +# ~33.6GB, which does NOT fit the 32GB R9700 on its own. docker-compose.yml +# now runs --cache-type-k/v q8_0, which roughly halves KV memory (~8GB at +# this size) — total ~25.6GB, ~6GB headroom, the same footprint the old +# 131072 fp16 setting used. See docs/research/qwen3.8-27b-quant.md. LLAMA_CTX_SIZE=262144 # Concurrent request slots. Was implicitly 4 (llama.cpp's compiled-in # default) with no flag set — under concurrent subagent fan-out, 4 requests @@ -39,7 +36,9 @@ LLAMA_CTX_SIZE=262144 # cancels the request (see issue-tracker notes on the timeout/cancel loop). # Dropped to 2 so each slot gets more compute and finishes prefill sooner; # raise back toward 4 if throughput (not latency) becomes the bottleneck -# instead. +# instead. Each slot gets LLAMA_CTX_SIZE / LLAMA_PARALLEL tokens of context — +# real sessions have hit ~66K tokens, so don't drop LLAMA_CTX_SIZE without +# checking that per-slot number stays comfortably above observed usage. LLAMA_PARALLEL=2 # --- Lazytainer --- diff --git a/docker-compose.yml b/docker-compose.yml index 51f82f4..6650c9d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,9 +18,11 @@ services: --host 0.0.0.0 --port 8080 --n-gpu-layers ${LLAMA_GPU_LAYERS:-999} - --ctx-size ${LLAMA_CTX_SIZE:-131072} + --ctx-size ${LLAMA_CTX_SIZE:-262144} --parallel ${LLAMA_PARALLEL:-2} --flash-attn on + --cache-type-k q8_0 + --cache-type-v q8_0 --jinja # No published host port: llama-server is reached only via the omniroute # gateway on the ai-stack docker network now — see issue #15. Its