From c5864beec93fc74c744c155b1fa6604d10a7c3a3 Mon Sep 17 00:00:00 2001 From: Haylan Date: Thu, 3 Sep 2026 06:35:48 +0200 Subject: [PATCH] feat(llama.cpp): raise default context to 128K, document RAM/SSD offload knobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LLAMA_CTX_SIZE default 65536 -> 131072: real usage was burning through 64K fast. ~25.6GB VRAM (17.6GB weights + ~8GB KV cache) on the 32GB R9700, ~6GB headroom — see docs/research/qwen3.8-27b-quant.md for the full table. Also documents LLAMA_GPU_LAYERS as the RAM-offload knob for this dense model (no separate RAM-offload flag exists in llama.cpp, and --n-cpu-moe/ --cpu-moe/--override-tensor "exps" are MoE-only, no-ops here), and that there's no explicit SSD offload tier to enable — llama.cpp's default mmap already falls back to disk implicitly if GPU+RAM run out. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01FCAUsjGNSoJTtK8hyLKg5m --- .env.example | 24 ++++++++++++++++++++---- docker-compose.yml | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 83db9b0..7a91a5b 100644 --- a/.env.example +++ b/.env.example @@ -6,11 +6,27 @@ # --- llama.cpp / model --- LLAMA_MODEL_FILE=Qwen3.8-27B-UD-Q4_K_XL.gguf +# 999 = every layer on GPU (this model is dense, not MoE, and already fits +# fully in 32GB VRAM — see docs/research/qwen3.8-27b-quant.md). Lower this +# to leave that many fewer layers on GPU and push the rest to CPU/system RAM +# if something else is contending for VRAM — llama.cpp has no separate +# "RAM offload" flag, --n-gpu-layers *is* the RAM-offload knob for a dense +# model. Don't reach for --n-cpu-moe/--cpu-moe/--override-tensor "exps" — +# those target Mixture-of-Experts models (e.g. Qwen3.8-2.4T-A95B), not this +# one, and are no-ops here. +# There's no separate "then SSD" tier to enable either: llama.cpp mmaps the +# model file by default (no --no-mmap here), so if GPU+RAM ever can't hold +# the working set, the OS pages the rest in from disk automatically — an +# implicit, slow last resort, not a config knob. An explicit tiered SSD +# offload has been an open llama.cpp feature request since 2025 (still +# unimplemented): https://github.com/ggml-org/llama.cpp/discussions/12507 LLAMA_GPU_LAYERS=999 -# 65536 (64K) fits comfortably in 32GB VRAM alongside the model weights. -# Raise toward 131072 if you need more context; see docs/research/qwen3.8-27b-quant.md -# for the VRAM math at larger context sizes. -LLAMA_CTX_SIZE=65536 +# 131072 (128K): ~25.6GB (17.6GB weights + ~8GB KV cache) on the 32GB +# R9700, ~6GB headroom — see docs/research/qwen3.8-27b-quant.md for the full +# table (64K only used ~19.6GB/~12GB headroom, but real usage was burning +# through 64K fast). If headroom gets tight, quantize the KV cache instead +# of dropping context: --cache-type-k/v q8_0 roughly halves it. +LLAMA_CTX_SIZE=131072 # --- Open WebUI --- WEBUI_PORT=8008 diff --git a/docker-compose.yml b/docker-compose.yml index f15e420..96bee97 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,7 +18,7 @@ services: --host 0.0.0.0 --port 8080 --n-gpu-layers ${LLAMA_GPU_LAYERS:-999} - --ctx-size ${LLAMA_CTX_SIZE:-65536} + --ctx-size ${LLAMA_CTX_SIZE:-131072} --jinja # No published host port: llama-server is reached only via the litellm # proxy on the ai-stack docker network now — see issue #15. Its -- 2.54.0