feat: add llama-server-fast, a small non-thinking classifier model

Second, always-resident llama.cpp instance (Qwen3-4B-Instruct-2507,
Q8_0 GGUF, ~5GB VRAM) alongside the existing Qwen3.8-27B instance, for
use as qwen-code CLI's Auto Mode classifier fastModel. Model choice
researched in docs/research/fast-model-choice.md: architecturally
non-thinking (unlike Qwen3-1.7B/0.6B), --reasoning off added
defensively per a known (closed) llama.cpp misdetection bug.

- docker-compose.yml: llama-server-fast + downloader-fast services,
  omniroute depends_on updated
- .env.example: LLAMA_FAST_* vars
- scripts/update.sh: runs the new downloader profile

Refs #44

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MrnMEdzeQzqZE5soVEXPCx
This commit is contained in:
2026-09-06 20:24:59 +02:00
co-authored by Claude-Bot
parent 71c9003bd8
commit d984c10835
4 changed files with 297 additions and 0 deletions
+58
View File
@@ -39,6 +39,46 @@ services:
- "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
- "lazytainer.group.llamaserver.minPacketThreshold=2"
llama-server-fast:
image: ghcr.io/ggml-org/llama.cpp:server-rocm
container_name: llama-server-fast
devices:
- /dev/kfd
- /dev/dri
group_add:
- video
- render
security_opt:
- seccomp=unconfined
ipc: host
volumes:
- models:/models
command: >
-m /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
--host 0.0.0.0
--port 8080
--n-gpu-layers ${LLAMA_FAST_GPU_LAYERS:-999}
--ctx-size ${LLAMA_FAST_CTX_SIZE:-8192}
--parallel ${LLAMA_FAST_PARALLEL:-2}
--flash-attn on
--cache-type-k q8_0
--cache-type-v q8_0
--reasoning off
--jinja
# Second, always-resident llama.cpp instance — small non-thinking model
# used as qwen-code's Auto Mode classifier fastModel, alongside the main
# 27B instance above. See docs/research/fast-model-choice.md and #44.
# Same ai-stack-only pattern as llama-server: no published host port.
expose:
- "8080"
restart: unless-stopped
networks: [ai-stack]
labels:
- "lazytainer.group.llamaserverfast.sleepMethod=stop"
- "lazytainer.group.llamaserverfast.ports=8080"
- "lazytainer.group.llamaserverfast.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
- "lazytainer.group.llamaserverfast.minPacketThreshold=2"
# ponytail: one-off downloader, not a standing service — run via
# `docker compose --profile tools run --rm downloader`. Folded into
# scripts/update.sh, which runs this every time; the `test -f` guard is
@@ -60,6 +100,22 @@ services:
curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
# Same test -f guard pattern as downloader above — fetches the second,
# smaller model for llama-server-fast. See issue #44.
downloader-fast:
image: curlimages/curl:latest
profiles: ["tools"]
user: root
volumes:
- models:/models
entrypoint: ["sh", "-c"]
command:
- >
test -f /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} &&
echo "already downloaded, skipping" ||
curl -L --fail --create-dirs -o /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
# Local image generation — see issue #38 (wayfinder map). yurisasc's image
# is gfx1201-tuned specifically (R9700's arch), unlike the official/AMD
# ComfyUI image which doesn't pin RDNA4 support — see
@@ -118,6 +174,8 @@ services:
depends_on:
llama-server:
condition: service_started
llama-server-fast:
condition: service_started
volumes:
- omniroute-data:/app/data
env_file: .env