diff --git a/.env.example b/.env.example index 6922471..df4c744 100644 --- a/.env.example +++ b/.env.example @@ -43,10 +43,10 @@ LLAMA_CTX_SIZE=262144 # comfortably above observed usage. LLAMA_PARALLEL=2 -# Dedicated CPU-only backend for qwen-code's tool-call harmfulness classifier -# (fastModel in ~/.qwen/settings.json) — see docker-compose.yml's -# qwen-classifier service comment for the why and the RAM math. -LLAMA_CLASSIFIER_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf +# Dedicated GPU-resident backend for qwen-code's tool-call harmfulness +# classifier (fastModel in ~/.qwen/settings.json) — see docker-compose.yml's +# qwen-classifier service comment for the why and the VRAM/context math. +LLAMA_CLASSIFIER_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf # --- Lazytainer --- # Seconds of inactivity before llama-server is stopped. 900 = 15 min. diff --git a/docker-compose.yml b/docker-compose.yml index e2db7b8..ace57f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,31 +50,50 @@ services: # (fastModel in ~/.qwen/settings.json). Was aliased onto llama-server's own # 27B connection — every classification call then queued behind whatever # heavy generation was already running on that model's 2 GPU slots (issue - # tracker: OmniRoute semaphore/pr-agent investigation). CPU-only, own - # process, own queue: structurally can't contend with llama-server for a - # GPU slot. Needs >=131072 ctx (qwen-code requirement); Qwen3-4B-Instruct-2507 - # is the smallest Qwen3 that supports that natively (262144) without - # RoPE-scaling — the smaller 0.6B/1.7B/4B (non-2507) models only go to - # 40960. Reuses a Q8_K_XL GGUF already sitting in the models volume from - # something earlier (better weight quality than the Q4_K_M-class file - # originally specced here, no download needed). KV cache dropped to - # q4_0/q4_0 to compensate: full-context q8_0/q8_0 (~9.6GiB) on top of the - # larger Q8 weights (~4.7GiB) left too little slack against the rest of - # the stack (omniroute's 10g mem_limit, qdrant, neo4j) on gameserver's - # 31GiB total RAM; q4_0/q4_0 (~5.1GiB) + weights (~4.7GiB) ≈ 9.8GiB leaves - # comfortable headroom instead. Weight precision matters more than KV - # precision for a classification task, so this trade favors the weights. + # tracker: OmniRoute semaphore/pr-agent investigation). + # + # Tried CPU-only first (own process avoids the GPU queue entirely) — too + # slow in practice: real classification calls blew past OmniRoute's 60s + # timeout and retry-looped (504→499→504...). Moved to GPU instead. + # + # Context sizing: qwen-code's classifier transcript is hard-capped in its + # own source (MAX_TRANSCRIPT_MESSAGES=40, MAX_HISTORICAL_ACTION_CHARS=4000 + # per message, packages/core/src/permissions/classifier-transcript.ts) — + # worst case is ~40-50K tokens, nowhere near the 131072 originally set in + # settings.json (that number was copied from the main model's entry, not + # a real qwen-code requirement). 65536 ctx gives ~1.5x margin over that + # worst case. Qwen3-4B-Instruct-2507 is still the model choice — smallest + # Qwen3 with long native context (262144) without RoPE-scaling, in case + # that margin ever needs to grow. + # + # VRAM: Q4_K_XL weights (~2.37GiB) + q4_0/q4_0 KV at 65536 ctx (~2.53GiB) + # ≈ 4.9GiB, fits fully on GPU (--n-gpu-layers 999) inside the ~6.1GiB free + # on the R9700 after llama-server's 27B model, with ~1.2GiB headroom. + # GPU_MAX_HW_QUEUES=1 carried over from llama-server's comment above — + # same ROCm/ROCm#5706 clock-pinning bug applies now that two HIP contexts + # (this + llama-server) share the card. qwen-classifier: - image: ghcr.io/ggml-org/llama.cpp:server + image: ghcr.io/ggml-org/llama.cpp:server-rocm container_name: qwen-classifier + devices: + - /dev/kfd + - /dev/dri + group_add: + - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}" + - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}" + security_opt: + - seccomp=unconfined + ipc: host + environment: + - GPU_MAX_HW_QUEUES=1 volumes: - models:/models command: > - -m /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} + -m /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf} --host 0.0.0.0 --port 8080 - --n-gpu-layers 0 - --ctx-size 131072 + --n-gpu-layers 999 + --ctx-size 65536 --parallel 1 --cache-type-k q4_0 --cache-type-v q4_0