feat: add llama-server-fast, a small non-thinking classifier model
Second, always-resident llama.cpp instance (Qwen3-4B-Instruct-2507, Q8_0 GGUF, ~5GB VRAM) alongside the existing Qwen3.8-27B instance, for use as qwen-code CLI's Auto Mode classifier fastModel. Model choice researched in docs/research/fast-model-choice.md: architecturally non-thinking (unlike Qwen3-1.7B/0.6B), --reasoning off added defensively per a known (closed) llama.cpp misdetection bug. - docker-compose.yml: llama-server-fast + downloader-fast services, omniroute depends_on updated - .env.example: LLAMA_FAST_* vars - scripts/update.sh: runs the new downloader profile Refs #44 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MrnMEdzeQzqZE5soVEXPCx
This commit is contained in:
@@ -39,6 +39,46 @@ services:
|
||||
- "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
|
||||
- "lazytainer.group.llamaserver.minPacketThreshold=2"
|
||||
|
||||
llama-server-fast:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-rocm
|
||||
container_name: llama-server-fast
|
||||
devices:
|
||||
- /dev/kfd
|
||||
- /dev/dri
|
||||
group_add:
|
||||
- video
|
||||
- render
|
||||
security_opt:
|
||||
- seccomp=unconfined
|
||||
ipc: host
|
||||
volumes:
|
||||
- models:/models
|
||||
command: >
|
||||
-m /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
|
||||
--host 0.0.0.0
|
||||
--port 8080
|
||||
--n-gpu-layers ${LLAMA_FAST_GPU_LAYERS:-999}
|
||||
--ctx-size ${LLAMA_FAST_CTX_SIZE:-8192}
|
||||
--parallel ${LLAMA_FAST_PARALLEL:-2}
|
||||
--flash-attn on
|
||||
--cache-type-k q8_0
|
||||
--cache-type-v q8_0
|
||||
--reasoning off
|
||||
--jinja
|
||||
# Second, always-resident llama.cpp instance — small non-thinking model
|
||||
# used as qwen-code's Auto Mode classifier fastModel, alongside the main
|
||||
# 27B instance above. See docs/research/fast-model-choice.md and #44.
|
||||
# Same ai-stack-only pattern as llama-server: no published host port.
|
||||
expose:
|
||||
- "8080"
|
||||
restart: unless-stopped
|
||||
networks: [ai-stack]
|
||||
labels:
|
||||
- "lazytainer.group.llamaserverfast.sleepMethod=stop"
|
||||
- "lazytainer.group.llamaserverfast.ports=8080"
|
||||
- "lazytainer.group.llamaserverfast.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
|
||||
- "lazytainer.group.llamaserverfast.minPacketThreshold=2"
|
||||
|
||||
# ponytail: one-off downloader, not a standing service — run via
|
||||
# `docker compose --profile tools run --rm downloader`. Folded into
|
||||
# scripts/update.sh, which runs this every time; the `test -f` guard is
|
||||
@@ -60,6 +100,22 @@ services:
|
||||
curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
|
||||
https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
|
||||
|
||||
# Same test -f guard pattern as downloader above — fetches the second,
|
||||
# smaller model for llama-server-fast. See issue #44.
|
||||
downloader-fast:
|
||||
image: curlimages/curl:latest
|
||||
profiles: ["tools"]
|
||||
user: root
|
||||
volumes:
|
||||
- models:/models
|
||||
entrypoint: ["sh", "-c"]
|
||||
command:
|
||||
- >
|
||||
test -f /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} &&
|
||||
echo "already downloaded, skipping" ||
|
||||
curl -L --fail --create-dirs -o /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
|
||||
https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
|
||||
|
||||
# Local image generation — see issue #38 (wayfinder map). yurisasc's image
|
||||
# is gfx1201-tuned specifically (R9700's arch), unlike the official/AMD
|
||||
# ComfyUI image which doesn't pin RDNA4 support — see
|
||||
@@ -118,6 +174,8 @@ services:
|
||||
depends_on:
|
||||
llama-server:
|
||||
condition: service_started
|
||||
llama-server-fast:
|
||||
condition: service_started
|
||||
volumes:
|
||||
- omniroute-data:/app/data
|
||||
env_file: .env
|
||||
|
||||
Reference in New Issue
Block a user