feat: remove llama-server-fast (Qwen3-4B classifier model)
Drops the second always-resident llama.cpp instance, its downloader, and the omniroute depends_on entry. Also strips the now-dead LLAMA_FAST_* block from .env.example and the stale VRAM-budget comment in scripts/switch-model.sh that assumed this service was always up. Note: this was qwen-code's Auto Mode Stage 1 classifier (fastModel) — see docs/research/fast-model-choice.md and issue #44. Auto Mode will lose that classifier until/unless it's reconfigured to route elsewhere or fall back to prompt-only classification. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DjhxWirQepKFEQj1huXNJR
This commit is contained in:
+4
-67
@@ -15,8 +15,7 @@ services:
|
||||
ipc: host
|
||||
# Caps this process's HIP hardware-queue allocation — works around
|
||||
# ROCm/ROCm#5706 (GPU pinned at 100%/boost-clock whenever two
|
||||
# concurrent HIP contexts touch this card, confirmed on real hardware
|
||||
# against llama-server-fast below). See the research doc above.
|
||||
# concurrent HIP contexts touch this card). See the research doc above.
|
||||
environment:
|
||||
- GPU_MAX_HW_QUEUES=1
|
||||
volumes:
|
||||
@@ -47,49 +46,6 @@ services:
|
||||
- "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
|
||||
- "lazytainer.group.llamaserver.minPacketThreshold=2"
|
||||
|
||||
llama-server-fast:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-rocm
|
||||
container_name: llama-server-fast
|
||||
devices:
|
||||
- /dev/kfd
|
||||
- /dev/dri
|
||||
group_add:
|
||||
- "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
|
||||
- "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
|
||||
security_opt:
|
||||
- seccomp=unconfined
|
||||
ipc: host
|
||||
# See llama-server's identical setting above — same fix, same bug.
|
||||
environment:
|
||||
- GPU_MAX_HW_QUEUES=1
|
||||
volumes:
|
||||
- models:/models
|
||||
command: >
|
||||
-m /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
|
||||
--host 0.0.0.0
|
||||
--port 8080
|
||||
--n-gpu-layers ${LLAMA_FAST_GPU_LAYERS:-999}
|
||||
--ctx-size ${LLAMA_FAST_CTX_SIZE:-8192}
|
||||
--parallel ${LLAMA_FAST_PARALLEL:-1}
|
||||
--flash-attn on
|
||||
--cache-type-k q8_0
|
||||
--cache-type-v q8_0
|
||||
--reasoning off
|
||||
--jinja
|
||||
# Second, always-resident llama.cpp instance — small non-thinking model
|
||||
# used as qwen-code's Auto Mode classifier fastModel, alongside the main
|
||||
# 27B instance above. See docs/research/fast-model-choice.md and #44.
|
||||
# Same ai-stack-only pattern as llama-server: no published host port.
|
||||
expose:
|
||||
- "8080"
|
||||
restart: unless-stopped
|
||||
networks: [ai-stack]
|
||||
labels:
|
||||
- "lazytainer.group.llamaserverfast.sleepMethod=stop"
|
||||
- "lazytainer.group.llamaserverfast.ports=8080"
|
||||
- "lazytainer.group.llamaserverfast.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
|
||||
- "lazytainer.group.llamaserverfast.minPacketThreshold=2"
|
||||
|
||||
# ponytail: one-off downloader, not a standing service — run via
|
||||
# `docker compose --profile tools run --rm downloader`. Folded into
|
||||
# scripts/update.sh, which runs this every time; the `test -f` guard is
|
||||
@@ -111,26 +67,9 @@ services:
|
||||
curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
|
||||
https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
|
||||
|
||||
# Same test -f guard pattern as downloader above — fetches the second,
|
||||
# smaller model for llama-server-fast. See issue #44.
|
||||
downloader-fast:
|
||||
image: curlimages/curl:latest
|
||||
profiles: ["tools"]
|
||||
user: root
|
||||
volumes:
|
||||
- models:/models
|
||||
entrypoint: ["sh", "-c"]
|
||||
command:
|
||||
- >
|
||||
test -f /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} &&
|
||||
echo "already downloaded, skipping" ||
|
||||
curl -L --fail --create-dirs -o /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
|
||||
https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
|
||||
|
||||
# Fetches the three Qwen-Image FP8 files ComfyUI needs (diffusion model,
|
||||
# text encoder, VAE) — same test -f guard pattern as downloader/
|
||||
# downloader-fast above. See docs/research/image-generation-model-choice.md
|
||||
# and issue #42.
|
||||
# text encoder, VAE) — same test -f guard pattern as downloader above.
|
||||
# See docs/research/image-generation-model-choice.md and issue #42.
|
||||
#
|
||||
# ponytail: target paths assume ComfyUI's standard models/ layout under
|
||||
# BASE_STORAGE_PATH (/storage) — same "not independently confirmed against
|
||||
@@ -184,7 +123,7 @@ services:
|
||||
# This image also wants GID env vars directly (its own README asks
|
||||
# for both these and group_add above) — same HOST_VIDEO_GID/
|
||||
# HOST_RENDER_GID resolved by scripts/update.sh, shared with
|
||||
# llama-server/llama-server-fast now instead of comfyui-only vars.
|
||||
# llama-server now instead of comfyui-only vars.
|
||||
- PUID=${COMFYUI_PUID}
|
||||
- PGID=${COMFYUI_PGID}
|
||||
- VIDEO_GID=${HOST_VIDEO_GID}
|
||||
@@ -220,8 +159,6 @@ services:
|
||||
depends_on:
|
||||
llama-server:
|
||||
condition: service_started
|
||||
llama-server-fast:
|
||||
condition: service_started
|
||||
volumes:
|
||||
- omniroute-data:/app/data
|
||||
env_file: .env
|
||||
|
||||
Reference in New Issue
Block a user