.env.example

@@ -6,11 +6,27 @@
 
 # --- llama.cpp / model ---
 LLAMA_MODEL_FILE=Qwen3.8-27B-UD-Q4_K_XL.gguf
+# 999 = every layer on GPU (this model is dense, not MoE, and already fits
+# fully in 32GB VRAM — see docs/research/qwen3.8-27b-quant.md). Lower this
+# to leave that many fewer layers on GPU and push the rest to CPU/system RAM
+# if something else is contending for VRAM — llama.cpp has no separate
+# "RAM offload" flag, --n-gpu-layers *is* the RAM-offload knob for a dense
+# model. Don't reach for --n-cpu-moe/--cpu-moe/--override-tensor "exps" —
+# those target Mixture-of-Experts models (e.g. Qwen3.8-2.4T-A95B), not this
+# one, and are no-ops here.
+# There's no separate "then SSD" tier to enable either: llama.cpp mmaps the
+# model file by default (no --no-mmap here), so if GPU+RAM ever can't hold
+# the working set, the OS pages the rest in from disk automatically — an
+# implicit, slow last resort, not a config knob. An explicit tiered SSD
+# offload has been an open llama.cpp feature request since 2025 (still
+# unimplemented): https://github.com/ggml-org/llama.cpp/discussions/12507
 LLAMA_GPU_LAYERS=999
-# 65536 (64K) fits comfortably in 32GB VRAM alongside the model weights.
+# 131072 (128K): ~25.6GB (17.6GB weights + ~8GB KV cache) on the 32GB
-# Raise toward 131072 if you need more context; see docs/research/qwen3.8-27b-quant.md
+# R9700, ~6GB headroom — see docs/research/qwen3.8-27b-quant.md for the full
-# for the VRAM math at larger context sizes.
+# table (64K only used ~19.6GB/~12GB headroom, but real usage was burning
-LLAMA_CTX_SIZE=65536
+# through 64K fast). If headroom gets tight, quantize the KV cache instead
+# of dropping context: --cache-type-k/v q8_0 roughly halves it.
+LLAMA_CTX_SIZE=131072
 
 # --- Open WebUI ---
 WEBUI_PORT=8008

docker-compose.yml

@@ -18,7 +18,7 @@ services:
       --host 0.0.0.0
       --port 8080
       --n-gpu-layers ${LLAMA_GPU_LAYERS:-999}
-      --ctx-size ${LLAMA_CTX_SIZE:-65536}
+      --ctx-size ${LLAMA_CTX_SIZE:-131072}
       --jinja
     # No published host port: llama-server is reached only via the litellm
     # proxy on the ai-stack docker network now — see issue #15. Its