From ba9ace6f714c602c22f8bf75766dbc29550809ae Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Wed, 9 Sep 2026 17:02:44 +0200 Subject: [PATCH] fix(llm): shrink qwen-classifier's compute buffer for real VRAM headroom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured live: full-offload weights+KV (~4.9GiB estimate) actually used ~5.85GiB, leaving only ~700MB free on the R9700 — too tight, real OOM risk for either GPU process. The gap was compute-buffer/graph overhead the naive estimate didn't account for. Drop --batch-size/--ubatch-size well below llama-server's defaults (2048/512) to shrink it — a single-request classifier has no batching throughput to lose. Co-Authored-By: Claude Sonnet 5 --- docker-compose.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ace57f9..81305d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,12 +66,17 @@ services: # Qwen3 with long native context (262144) without RoPE-scaling, in case # that margin ever needs to grow. # - # VRAM: Q4_K_XL weights (~2.37GiB) + q4_0/q4_0 KV at 65536 ctx (~2.53GiB) - # ≈ 4.9GiB, fits fully on GPU (--n-gpu-layers 999) inside the ~6.1GiB free - # on the R9700 after llama-server's 27B model, with ~1.2GiB headroom. - # GPU_MAX_HW_QUEUES=1 carried over from llama-server's comment above — - # same ROCm/ROCm#5706 clock-pinning bug applies now that two HIP contexts - # (this + llama-server) share the card. + # VRAM: weights+KV math (~4.9GiB) predicted comfortable headroom in the + # ~6.1GiB free on the R9700, but measured live it actually used ~5.85GiB — + # left only ~700MB free, too tight (compute-buffer/graph overhead the + # naive weights+KV estimate didn't account for). --batch-size/--ubatch-size + # dropped well below llama-server's defaults (2048/512) to shrink that + # buffer — this classifier processes one request at a time, no batching + # throughput to lose. --n-gpu-layers 999 (full offload) kept since the + # ~1.5GB freed here should cover it; re-verify with rocm-smi after deploy + # before trusting this number again. GPU_MAX_HW_QUEUES=1 carried over from + # llama-server's comment above — same ROCm/ROCm#5706 clock-pinning bug + # applies now that two HIP contexts (this + llama-server) share the card. qwen-classifier: image: ghcr.io/ggml-org/llama.cpp:server-rocm container_name: qwen-classifier @@ -95,6 +100,8 @@ services: --n-gpu-layers 999 --ctx-size 65536 --parallel 1 + --batch-size 512 + --ubatch-size 128 --cache-type-k q4_0 --cache-type-v q4_0 --jinja