From 20cc0bcc708a50d14d4697ae12db3841d0247a86 Mon Sep 17 00:00:00 2001
From: ArthurErlich <a.erlich@live.de>
Date: Wed, 9 Sep 2026 16:37:54 +0200
Subject: [PATCH] fix(llm): point qwen-classifier at the Q8 GGUF already on
 disk
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Q4_K_M-class file this originally specced didn't exist yet on
gameserver (classifier crash-looped: "No such file or directory").
A Q8_K_XL GGUF for the same model was already sitting in the models
volume from something earlier — point at that instead of downloading a
new file, and drop the KV cache quant to q4_0/q4_0 to keep total RAM
comfortable now that the weights are the larger Q8 variant.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 .env.example       |  6 ++----
 docker-compose.yml | 19 ++++++++++++-------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/.env.example b/.env.example
index 6601878..6922471 100644
--- a/.env.example
+++ b/.env.example
@@ -45,10 +45,8 @@ LLAMA_PARALLEL=2
 
 # Dedicated CPU-only backend for qwen-code's tool-call harmfulness classifier
 # (fastModel in ~/.qwen/settings.json) — see docker-compose.yml's
-# qwen-classifier service comment for the why. 131072 ctx / q8_0 KV / Q4_K_M-
-# class weights ≈ 12GiB, fits gameserver's ~17GiB free system RAM with room
-# to spare.
-LLAMA_CLASSIFIER_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf
+# qwen-classifier service comment for the why and the RAM math.
+LLAMA_CLASSIFIER_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf
 
 # --- Lazytainer ---
 # Seconds of inactivity before llama-server is stopped. 900 = 15 min.
diff --git a/docker-compose.yml b/docker-compose.yml
index c09ecee..e2db7b8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -55,24 +55,29 @@ services:
   # GPU slot. Needs >=131072 ctx (qwen-code requirement); Qwen3-4B-Instruct-2507
   # is the smallest Qwen3 that supports that natively (262144) without
   # RoPE-scaling — the smaller 0.6B/1.7B/4B (non-2507) models only go to
-  # 40960. Sized for gameserver's ~17GiB free RAM: q8_0/q8_0 KV at full
-  # 131072 ctx is ~9.8GiB + ~2.3GiB Q4_K_M-class weights ≈ 12GiB, comfortable
-  # headroom, and better KV quality than the q4_0 that would've been needed
-  # to squeeze this onto the GPU's ~6GiB free VRAM instead.
+  # 40960. Reuses a Q8_K_XL GGUF already sitting in the models volume from
+  # something earlier (better weight quality than the Q4_K_M-class file
+  # originally specced here, no download needed). KV cache dropped to
+  # q4_0/q4_0 to compensate: full-context q8_0/q8_0 (~9.6GiB) on top of the
+  # larger Q8 weights (~4.7GiB) left too little slack against the rest of
+  # the stack (omniroute's 10g mem_limit, qdrant, neo4j) on gameserver's
+  # 31GiB total RAM; q4_0/q4_0 (~5.1GiB) + weights (~4.7GiB) ≈ 9.8GiB leaves
+  # comfortable headroom instead. Weight precision matters more than KV
+  # precision for a classification task, so this trade favors the weights.
   qwen-classifier:
     image: ghcr.io/ggml-org/llama.cpp:server
     container_name: qwen-classifier
     volumes:
       - models:/models
     command: >
-      -m /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}
+      -m /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
       --host 0.0.0.0
       --port 8080
       --n-gpu-layers 0
       --ctx-size 131072
       --parallel 1
-      --cache-type-k q8_0
-      --cache-type-v q8_0
+      --cache-type-k q4_0
+      --cache-type-v q4_0
       --jinja
     expose:
       - "8080"