diff --git a/.env.example b/.env.example index 5108412..5668498 100644 --- a/.env.example +++ b/.env.example @@ -105,34 +105,6 @@ COMFYUI_PGID= HOST_VIDEO_GID= HOST_RENDER_GID= -# --- llama.cpp / fast model (second, always-resident instance — see -# docs/research/fast-model-choice.md and issue #44) --- -# Qwen3-4B-Instruct-2507: architecturally non-thinking (never emits -# blocks, unlike Qwen3-1.7B/0.6B which need a per-call toggle) — -# picked specifically so it stays fast enough for qwen-code's Auto Mode -# classifier (Stage 1 wants ~300ms). Same publisher (unsloth) as the main -# model for consistency. -LLAMA_FAST_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf -# Same reasoning as LLAMA_GPU_LAYERS above — full GPU offload, this model -# is dense too. -LLAMA_FAST_GPU_LAYERS=999 -# --ctx-size is the TOTAL across every LLAMA_FAST_PARALLEL slot, not per -# request — same halving already called out for the main model above. -# Was PARALLEL=2, silently halving this to 4096/slot — too small: a real -# classifier call (hints + environment + recent tool-call history) hit -# "exceeds the available context size (4096 tokens)" in practice, which -# qwen-code surfaces as "Auto Mode couldn't classify this action -# (Classifier stage 1 unavailable)" — see issue #5. Fixed by dropping to -# a single slot instead of raising ctx-size (no extra VRAM, and this -# service doesn't need concurrent classifier calls the way the main -# model needs concurrent chat sessions) — the full 8192 now goes to the -# one slot. If hints.allow/softDeny/hardDeny ever approach their -# 50-entries-each ceiling, raise LLAMA_FAST_CTX_SIZE instead — qwen-code -# caps those at 200 chars x 150 entries plus 40,000 chars of -# historical-action context, which can exceed 8192 tokens worst-case. -LLAMA_FAST_CTX_SIZE=8192 -LLAMA_FAST_PARALLEL=1 - # --- ComfyUI diffusion model (Qwen-Image, FP8 — see docs/research/ # image-generation-model-choice.md and issue #42) --- # Three files: diffusion weights, text encoder, VAE — all from the official diff --git a/docker-compose.yml b/docker-compose.yml index 5c0ed1d..7aa40eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,8 +15,7 @@ services: ipc: host # Caps this process's HIP hardware-queue allocation — works around # ROCm/ROCm#5706 (GPU pinned at 100%/boost-clock whenever two - # concurrent HIP contexts touch this card, confirmed on real hardware - # against llama-server-fast below). See the research doc above. + # concurrent HIP contexts touch this card). See the research doc above. environment: - GPU_MAX_HW_QUEUES=1 volumes: @@ -47,49 +46,6 @@ services: - "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}" - "lazytainer.group.llamaserver.minPacketThreshold=2" - llama-server-fast: - image: ghcr.io/ggml-org/llama.cpp:server-rocm - container_name: llama-server-fast - devices: - - /dev/kfd - - /dev/dri - group_add: - - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}" - - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}" - security_opt: - - seccomp=unconfined - ipc: host - # See llama-server's identical setting above — same fix, same bug. - environment: - - GPU_MAX_HW_QUEUES=1 - volumes: - - models:/models - command: > - -m /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} - --host 0.0.0.0 - --port 8080 - --n-gpu-layers ${LLAMA_FAST_GPU_LAYERS:-999} - --ctx-size ${LLAMA_FAST_CTX_SIZE:-8192} - --parallel ${LLAMA_FAST_PARALLEL:-1} - --flash-attn on - --cache-type-k q8_0 - --cache-type-v q8_0 - --reasoning off - --jinja - # Second, always-resident llama.cpp instance — small non-thinking model - # used as qwen-code's Auto Mode classifier fastModel, alongside the main - # 27B instance above. See docs/research/fast-model-choice.md and #44. - # Same ai-stack-only pattern as llama-server: no published host port. - expose: - - "8080" - restart: unless-stopped - networks: [ai-stack] - labels: - - "lazytainer.group.llamaserverfast.sleepMethod=stop" - - "lazytainer.group.llamaserverfast.ports=8080" - - "lazytainer.group.llamaserverfast.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}" - - "lazytainer.group.llamaserverfast.minPacketThreshold=2" - # ponytail: one-off downloader, not a standing service — run via # `docker compose --profile tools run --rm downloader`. Folded into # scripts/update.sh, which runs this every time; the `test -f` guard is @@ -111,26 +67,9 @@ services: curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} - # Same test -f guard pattern as downloader above — fetches the second, - # smaller model for llama-server-fast. See issue #44. - downloader-fast: - image: curlimages/curl:latest - profiles: ["tools"] - user: root - volumes: - - models:/models - entrypoint: ["sh", "-c"] - command: - - > - test -f /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} && - echo "already downloaded, skipping" || - curl -L --fail --create-dirs -o /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} - https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} - # Fetches the three Qwen-Image FP8 files ComfyUI needs (diffusion model, - # text encoder, VAE) — same test -f guard pattern as downloader/ - # downloader-fast above. See docs/research/image-generation-model-choice.md - # and issue #42. + # text encoder, VAE) — same test -f guard pattern as downloader above. + # See docs/research/image-generation-model-choice.md and issue #42. # # ponytail: target paths assume ComfyUI's standard models/ layout under # BASE_STORAGE_PATH (/storage) — same "not independently confirmed against @@ -184,7 +123,7 @@ services: # This image also wants GID env vars directly (its own README asks # for both these and group_add above) — same HOST_VIDEO_GID/ # HOST_RENDER_GID resolved by scripts/update.sh, shared with - # llama-server/llama-server-fast now instead of comfyui-only vars. + # llama-server now instead of comfyui-only vars. - PUID=${COMFYUI_PUID} - PGID=${COMFYUI_PGID} - VIDEO_GID=${HOST_VIDEO_GID} @@ -220,8 +159,6 @@ services: depends_on: llama-server: condition: service_started - llama-server-fast: - condition: service_started volumes: - omniroute-data:/app/data env_file: .env diff --git a/scripts/switch-model.sh b/scripts/switch-model.sh index 8aec6b6..1b7f1b1 100644 --- a/scripts/switch-model.sh +++ b/scripts/switch-model.sh @@ -11,12 +11,6 @@ # it never reliably sleeps a service on its own. A scripted swap always # knows which service should go up/down, so it doesn't need that heuristic. # -# llama-server-fast (the small classifier model, issue #44) is NOT part of -# this swap — it's meant to stay always-resident. Worst case with comfyui up -# is comfyui (~25GB, Qwen-Image FP8) + llama-server-fast (~5GB) ≈ 30GB, -# still under the 32GB card but tight — unverified on real hardware, check -# `docker compose ps` / VRAM usage after the first real swap. -# # OmniRoute may show the just-stopped provider as errored/offline in its # dashboard for up to CREDENTIAL_HEALTH_CHECK_INTERVAL (default 5 min) after # a swap — cosmetic, not a functional problem (see the research doc above).