From d984c1083587bcb319ab9079d2cf6070a70464ac Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Sun, 6 Sep 2026 20:24:59 +0200 Subject: [PATCH] feat: add llama-server-fast, a small non-thinking classifier model Second, always-resident llama.cpp instance (Qwen3-4B-Instruct-2507, Q8_0 GGUF, ~5GB VRAM) alongside the existing Qwen3.8-27B instance, for use as qwen-code CLI's Auto Mode classifier fastModel. Model choice researched in docs/research/fast-model-choice.md: architecturally non-thinking (unlike Qwen3-1.7B/0.6B), --reasoning off added defensively per a known (closed) llama.cpp misdetection bug. - docker-compose.yml: llama-server-fast + downloader-fast services, omniroute depends_on updated - .env.example: LLAMA_FAST_* vars - scripts/update.sh: runs the new downloader profile Refs #44 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01MrnMEdzeQzqZE5soVEXPCx --- .env.example | 17 +++ docker-compose.yml | 58 ++++++++ docs/research/fast-model-choice.md | 221 +++++++++++++++++++++++++++++ scripts/update.sh | 1 + 4 files changed, 297 insertions(+) create mode 100644 docs/research/fast-model-choice.md diff --git a/.env.example b/.env.example index 51cd867..0d36a98 100644 --- a/.env.example +++ b/.env.example @@ -96,3 +96,20 @@ COMFYUI_PUID= COMFYUI_PGID= COMFYUI_VIDEO_GID= COMFYUI_RENDER_GID= + +# --- llama.cpp / fast model (second, always-resident instance — see +# docs/research/fast-model-choice.md and issue #44) --- +# Qwen3-4B-Instruct-2507: architecturally non-thinking (never emits +# blocks, unlike Qwen3-1.7B/0.6B which need a per-call toggle) — +# picked specifically so it stays fast enough for qwen-code's Auto Mode +# classifier (Stage 1 wants ~300ms). Same publisher (unsloth) as the main +# model for consistency. +LLAMA_FAST_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf +# Same reasoning as LLAMA_GPU_LAYERS above — full GPU offload, this model +# is dense too. +LLAMA_FAST_GPU_LAYERS=999 +# Classifier transcripts are truncated/bounded by qwen-code itself (see its +# own Auto Mode docs) — no need for anywhere near the 27B's huge context. +# 8192 keeps this instance's KV cache negligible. +LLAMA_FAST_CTX_SIZE=8192 +LLAMA_FAST_PARALLEL=2 diff --git a/docker-compose.yml b/docker-compose.yml index b84bbc3..1bcda03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -39,6 +39,46 @@ services: - "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}" - "lazytainer.group.llamaserver.minPacketThreshold=2" + llama-server-fast: + image: ghcr.io/ggml-org/llama.cpp:server-rocm + container_name: llama-server-fast + devices: + - /dev/kfd + - /dev/dri + group_add: + - video + - render + security_opt: + - seccomp=unconfined + ipc: host + volumes: + - models:/models + command: > + -m /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} + --host 0.0.0.0 + --port 8080 + --n-gpu-layers ${LLAMA_FAST_GPU_LAYERS:-999} + --ctx-size ${LLAMA_FAST_CTX_SIZE:-8192} + --parallel ${LLAMA_FAST_PARALLEL:-2} + --flash-attn on + --cache-type-k q8_0 + --cache-type-v q8_0 + --reasoning off + --jinja + # Second, always-resident llama.cpp instance — small non-thinking model + # used as qwen-code's Auto Mode classifier fastModel, alongside the main + # 27B instance above. See docs/research/fast-model-choice.md and #44. + # Same ai-stack-only pattern as llama-server: no published host port. + expose: + - "8080" + restart: unless-stopped + networks: [ai-stack] + labels: + - "lazytainer.group.llamaserverfast.sleepMethod=stop" + - "lazytainer.group.llamaserverfast.ports=8080" + - "lazytainer.group.llamaserverfast.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}" + - "lazytainer.group.llamaserverfast.minPacketThreshold=2" + # ponytail: one-off downloader, not a standing service — run via # `docker compose --profile tools run --rm downloader`. Folded into # scripts/update.sh, which runs this every time; the `test -f` guard is @@ -60,6 +100,22 @@ services: curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} + # Same test -f guard pattern as downloader above — fetches the second, + # smaller model for llama-server-fast. See issue #44. + downloader-fast: + image: curlimages/curl:latest + profiles: ["tools"] + user: root + volumes: + - models:/models + entrypoint: ["sh", "-c"] + command: + - > + test -f /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} && + echo "already downloaded, skipping" || + curl -L --fail --create-dirs -o /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} + https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} + # Local image generation — see issue #38 (wayfinder map). yurisasc's image # is gfx1201-tuned specifically (R9700's arch), unlike the official/AMD # ComfyUI image which doesn't pin RDNA4 support — see @@ -118,6 +174,8 @@ services: depends_on: llama-server: condition: service_started + llama-server-fast: + condition: service_started volumes: - omniroute-data:/app/data env_file: .env diff --git a/docs/research/fast-model-choice.md b/docs/research/fast-model-choice.md new file mode 100644 index 0000000..4f5bac3 --- /dev/null +++ b/docs/research/fast-model-choice.md @@ -0,0 +1,221 @@ +# Which small model to run as the always-resident `fastModel` for qwen-code's Auto Mode classifier? + +**Date:** 2026-09-06 +**Budget:** ≤7GB VRAM, resident concurrently alongside the existing Qwen3.8-27B instance on the single +32GB R9700, via the same `llama.cpp:server-rocm` image already in `docker-compose.yml`. +**Answer: Qwen3-4B-Instruct-2507, Q8_0 GGUF (~4.3GB weights).** The prior quick pass's tentative pick +holds up under primary-source verification, for a more specific reason than "same tokenizer family": +it is the only strong candidate in the shortlist that is *architecturally* non-thinking (no `` +code path exists at all, vs. models that are thinking-by-default and rely on a per-call +`enable_thinking:false` toggle that llama.cpp does not cleanly expose). It does carry one directly +relevant, documented llama.cpp bug — but that bug is closed, has a one-flag workaround, and is +strictly less severe than the still-open Qwen3.5/Qwen3.8-lineage bugs already documented against the +27B model in this repo. + +## 1. What the classifier actually needs (grounding the requirement) + +Per qwen-code's own docs, Auto Mode's permission gate is a two-stage LLM classifier: + +- **Stage 1** — outputs only `{ shouldBlock: bool }`, ~300ms budget, thinking already disabled at the + request level. If `shouldBlock` is `false`, the action proceeds immediately. +- **Stage 2** — only runs when Stage 1 blocks; uses chain-of-thought review to downgrade false + positives, ~3-5s budget. +- Both stages use "your configured fast model (`/model --fast`)"; if none is configured, the full + session model is used instead — which is the current, too-slow state this second model is meant to + fix. + +Source: [Qwen Code docs — Auto Mode](https://qwenlm.github.io/qwen-code-docs/en/users/features/auto-mode/), +[QwenLM/qwen-code docs/users/features/auto-mode.md](https://github.com/QwenLM/qwen-code/blob/main/docs/users/features/auto-mode.md). + +A live qwen-code issue independently confirms the exact failure mode this repo already hit with the +27B model — a model *thinking* inside the classifier path is a first-order latency problem, not a +nice-to-have to tune later: + +> "for a latency-sensitive permission gate, thinking should be disabled in every stage" — enabling it +> "makes the review path slower and more expensive, which directly worsens the timeout problem." + +That issue (timeouts tripping on slow inference) was closed by a PR that both loosened the stage +timeout budgets *and* moved toward disabling thinking everywhere in the classifier. +Source: [QwenLM/qwen-code issue #4676](https://github.com/QwenLM/qwen-code/issues/4676). + +Takeaway for model selection: the request-level "don't think" instruction already exists in +qwen-code's own classifier code. What matters is whether the **model + llama.cpp combination actually +honors it reliably** — which is precisely where the 27B model failed (see +[`qwen3.8-27b-tool-calling.md`](qwen3.8-27b-tool-calling.md)) and where several shortlist candidates +have their own version of the same problem. + +## 2. Candidates evaluated against primary sources + +| Model | Params | GGUF size (quant) | Context | License | Thinking behavior | Tool-calling | Verdict | +|---|---|---|---|---|---|---|---| +| **Qwen3-4B-Instruct-2507** | 4B | Q4_K_M 2.5GB / **Q8_0 4.28GB** | 262,144 native | Apache 2.0 | **Non-thinking only** — model card states it "does not generate `` blocks in its output," full stop, no toggle needed | Yes, native `` format, BFCL-v3 61.9 | **Recommended** | +| Qwen3-1.7B | 1.7B | ~1.1GB (Q4_K_M, typical) | 32,768 | Apache 2.0 | Thinking **on by default**; needs `enable_thinking:false` per call | Yes | Rejected — see §3 | +| Qwen3-0.6B | 0.6B | ~0.4GB (Q4_K_M) | 32,768 | Apache 2.0 | Thinking on by default, same toggle issue as 1.7B | Yes, but weakest reasoning of the family | Rejected — undersized for reliability at this size, same toggle risk | +| Llama-3.2-3B-Instruct | 3B | ~2GB (Q4_K_M, typical) | 128K | Llama 3.2 Community License — commercial use allowed, but text/EU carve-out language and an explicit >700M-MAU re-licensing clause | No thinking mode | Not natively documented on the model card fetched (no tool-call format called out) | Deprioritized — license has more fine print than Apache 2.0 for no clear benefit here | +| Gemma-3-4b-it | 4B | Q4_K_M 2.49GB / Q8_0 4.13GB | 128K | Custom "Gemma" license (Google usage terms) | No documented thinking mode | Not documented on the model card fetched | Deprioritized — no confirmed native tool-calling story, non-Apache license | +| Phi-4-mini-instruct | 3.8B | Q4_K_M 2.49GB / Q8_0 4.08GB | 128K | **MIT** | Not a reasoning model (that's the separate Phi-4-mini-**reasoning** model); no `` tags by default | Yes — documented function-call format with dedicated tokens | Credible alternative — see §4 | +| SmolLM3-3B | 3B | Q4_K_M ~1.9GB (typical) | 128K (64K trained + YaRN) | Apache 2.0 | **Thinking on by default** (`enable_thinking`), toggled via system-prompt flags | Yes (XML or Python-style tool calls) | Rejected — same thinking-by-default risk as Qwen3-1.7B | +| Ministral-8B-Instruct-2410 | 8B | too large for budget at any useful quant with headroom | 128K | **Mistral Research License — commercial use requires contacting Mistral for a separate license** | Not documented as a reasoning model | Yes, documented function-calling with benchmark (31.6 vs Mistral-7B's 6.9) | Rejected — license restricts this repo's own dev-tooling use without contacting Mistral; also parameter count crowds the 7GB budget once Q8_0 + KV cache is counted | + +Sources (fetched directly from each model's own HF card / GGUF repo unless noted): +[Qwen/Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507), +[unsloth/Qwen3-4B-Instruct-2507-GGUF](https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF), +[Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B), +[Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B), +[meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct), +[google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it), +[bartowski/google_gemma-3-4b-it-GGUF](https://huggingface.co/bartowski/google_gemma-3-4b-it-GGUF), +[microsoft/Phi-4-mini-instruct](https://huggingface.co/microsoft/Phi-4-mini-instruct), +[bartowski/microsoft_Phi-4-mini-instruct-GGUF](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF), +[HuggingFaceTB/SmolLM3-3B](https://huggingface.co/HuggingFaceTB/SmolLM3-3B), +[mistralai/Ministral-8B-Instruct-2410](https://huggingface.co/mistralai/Ministral-8B-Instruct-2410). + +**Confidence note:** file sizes for Qwen3-1.7B/0.6B, Llama-3.2-3B, and SmolLM3-3B GGUF quants above are +typical/approximate — those repos weren't individually re-verified against a specific GGUF file tree +since all three families were eliminated on architectural grounds (§3) before size mattered. Sizes for +the two models actually compared head-to-head (Qwen3-4B-Instruct-2507, Phi-4-mini-instruct) and +Gemma-3-4b-it were pulled directly from each quantizer's own repo page. + +## 3. Why "thinking-by-default + per-call toggle" is disqualifying, not just a minor ding + +This is the deciding architectural distinction, and it's exactly the failure this second model exists +to avoid. Qwen's own llama.cpp docs page states the toggle problem directly: + +> "the hard switch implemented in the chat template is not exposed in llama.cpp" for controlling +> `enable_thinking` — the documented workaround is to supply "a custom chat template equivalent to +> always `enable_thinking=False`" via `--chat-template-file`. +Source: [Qwen — Run with llama.cpp](https://qwen.readthedocs.io/en/latest/run_locally/llama.cpp.html). + +That means for Qwen3-1.7B, Qwen3-0.6B, and SmolLM3-3B — all thinking-on-by-default — reliably +suppressing the reasoning phase in this llama.cpp/ROCm stack is not a request-body flag away; it needs +a hand-maintained custom chat template file, which is exactly the kind of fragile, easy-to-silently- +regress setup this task is trying to get away from (the 27B model's whole problem was reasoning_content +being consumed before the answer). Qwen3-4B-Instruct-2507 has no such toggle to maintain in the first +place — the model card states the non-thinking behavior as an unconditional property of the model, not +a configurable default that has to be forced correctly on every request. This is a stronger claim than +"same tokenizer family as the 27B" (the original quick-pass's reasoning) and is the actual basis for +the recommendation. + +## 4. The one documented risk specific to Qwen3-4B-Instruct-2507 — and why it doesn't change the pick + +llama.cpp has its own closed, dated bug where server builds around **b8429** (March 2026) +mis-detected Qwen3-Instruct-2507 models — the 4B included by name in the reporter's repro command — as +thinking models, routing tool-call output into `reasoning_content` instead of `tool_calls`: + +> "llama.cpp b8429 incorrectly detects Qwen3-Instruct-2507 models as thinking models (`thinking = 1`). +> This causes tool calls to be captured as `reasoning_content` instead of being parsed into the +> `tool_calls` array." + +The documented, confirmed-working workaround is a single server flag: + +``` +llama-server -hf unsloth/Qwen3-4B-Instruct-2507-GGUF:Q4_K_M --jinja --port 8222 --reasoning off +``` + +which restores `thinking = 0` and correct `finish_reason: tool_calls` output. The issue is **closed**. +Source: [ggml-org/llama.cpp issue #20809](https://github.com/ggml-org/llama.cpp/issues/20809). + +This is worth flagging honestly against the recommendation, but it's materially different from the +open, only-partially-fixed Qwen3.5/Qwen3.8-lineage parser bugs already documented in this repo's +[`qwen3.8-27b-tool-calling.md`](qwen3.8-27b-tool-calling.md) (issues #21158, #20837 — both open at time +of that research): this is a llama.cpp *server-side misdetection* bug with a one-flag fix, not an +unresolved upstream grammar/parser defect in the Qwen3.5 architecture family itself. Concretely: add +`--reasoning off` to this second llama-server instance's command regardless — it's a no-cost safety net +whether or not the current `ghcr.io/ggml-org/llama.cpp:server-rocm` build still has the bug, and it +directly targets the exact failure mode (reasoning_content eating the completion) that ruled out the +27B model for this role in the first place. + +## 5. VRAM math for the classifier role specifically + +Qwen3-4B-Instruct-2507 is a plain (non-hybrid) transformer — every layer is standard GQA attention, so +unlike the 27B model's Gated-DeltaNet hybrid, KV cache scales with *all* layers, not a fraction of them. +From the model's own `config.json`: + +- `num_hidden_layers`: 36, `num_key_value_heads`: 8, `head_dim`: 128 +Source: [Qwen/Qwen3-4B-Instruct-2507 config.json](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/raw/main/config.json). + +Per-token KV cache (fp16, both K and V): +`36 layers × 2 (K+V) × 8 kv_heads × 128 head_dim × 2 bytes = 144 KiB/token` + +The classifier transcript is bounded by design — qwen-code's own two-stage design keeps Stage 1 to a +`{shouldBlock}`-only judgment and Stage 2 to a chain-of-thought review of one blocked action, not an +open-ended agent session — so a context window in the low thousands of tokens is generous headroom, +not a tight fit: + +| Context | KV cache (fp16) | KV cache (q8_0, `--cache-type-k/v q8_0`) | Weights (Q8_0) | Total (q8_0 KV) | Headroom under 7GB | +|---|---|---|---|---|---| +| 4,096 tokens | ~0.56 GB | ~0.28 GB | 4.28 GB | **~4.56 GB** | ~2.4 GB | +| 8,192 tokens | ~1.13 GB | ~0.56 GB | 4.28 GB | **~4.84 GB** | ~2.2 GB | +| 32,768 tokens (generous ceiling) | ~4.5 GB | ~2.25 GB | 4.28 GB | **~6.53 GB** | ~0.5 GB (tight) | + +At any context length actually needed for a permission-gate classifier (thousands, not tens of +thousands, of tokens), Q8_0 weights plus q8_0 KV cache comfortably clears the 7GB ceiling with headroom +to spare for the compute buffer and batch overhead — matching the same `--cache-type-k q8_0 +--cache-type-v q8_0` pattern this repo already uses for the 27B instance. There's no need to drop to +Q4_K_M (2.5GB) unless a much larger classifier context is anticipated later; Q8_0 is the better default +here since it's a small model where quantization loss matters proportionally more, and the VRAM budget +comfortably affords the higher-precision quant. + +## 6. What would change the answer + +- **If Phi-4-mini-instruct's MIT license matters more than matching the 27B model's tokenizer/template + family**, it's a legitimate second choice: confirmed non-thinking by default, confirmed native + function-calling format, comparable Q8_0 size (4.08GB), and a license with zero commercial-use fine + print (vs. Apache 2.0's still-permissive but slightly more conditional terms). It wasn't picked + because it has no llama.cpp-specific tool-calling track record verified in this pass (no equivalent + to the issue #20809 workaround search done for it), so its actual reliability on this exact + `llama.cpp:server-rocm` stack is less directly evidenced than Qwen3-4B-Instruct-2507's. +- **If the classifier transcript ever needs to grow well past ~8K tokens routinely**, drop to Q4_K_M + (2.5GB) to keep well clear of the 7GB ceiling — the KV-cache math in §5 shows the crossover point. +- **If llama.cpp's #20809 misdetection turns out to still reproduce** on the exact + `ghcr.io/ggml-org/llama.cpp:server-rocm` build this repo pulls, the fix is the one-flag + `--reasoning off` workaround already confirmed in that issue — not a reason to pick a different + model, since every thinking-capable alternative in this shortlist has an equal-or-worse version of + the same class of bug with less clean workarounds (custom chat-template files, per §3). + +## Sources + +- [Qwen Code docs — Auto Mode](https://qwenlm.github.io/qwen-code-docs/en/users/features/auto-mode/) +- [QwenLM/qwen-code — docs/users/features/auto-mode.md](https://github.com/QwenLM/qwen-code/blob/main/docs/users/features/auto-mode.md) +- [QwenLM/qwen-code issue #4676](https://github.com/QwenLM/qwen-code/issues/4676) +- [Qwen/Qwen3-4B-Instruct-2507](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507) +- [Qwen/Qwen3-4B-Instruct-2507 config.json](https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507/raw/main/config.json) +- [unsloth/Qwen3-4B-Instruct-2507-GGUF](https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF) +- [Qwen — Run with llama.cpp](https://qwen.readthedocs.io/en/latest/run_locally/llama.cpp.html) +- [ggml-org/llama.cpp issue #20809](https://github.com/ggml-org/llama.cpp/issues/20809) +- [Qwen/Qwen3-1.7B](https://huggingface.co/Qwen/Qwen3-1.7B) +- [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B) +- [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) +- [google/gemma-3-4b-it](https://huggingface.co/google/gemma-3-4b-it) +- [bartowski/google_gemma-3-4b-it-GGUF](https://huggingface.co/bartowski/google_gemma-3-4b-it-GGUF) +- [microsoft/Phi-4-mini-instruct](https://huggingface.co/microsoft/Phi-4-mini-instruct) +- [bartowski/microsoft_Phi-4-mini-instruct-GGUF](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF) +- [HuggingFaceTB/SmolLM3-3B](https://huggingface.co/HuggingFaceTB/SmolLM3-3B) +- [mistralai/Ministral-8B-Instruct-2410](https://huggingface.co/mistralai/Ministral-8B-Instruct-2410) +- [docs/research/qwen3.8-27b-tool-calling.md](qwen3.8-27b-tool-calling.md) (this repo — cross-referenced + for the 27B model's own, still-open, tool-calling parser bugs) + +## Confidence/uncertainty summary + +- **High confidence:** Qwen3-4B-Instruct-2507's non-thinking-only status (direct model-card quote); + qwen-code's two-stage classifier timing/design and its use of `/model --fast` (direct docs quote); + the existence, exact symptom, and workaround of llama.cpp issue #20809 (direct issue quote); the + KV-cache architecture math (computed directly from the model's own `config.json`, same method as the + existing `qwen3.8-27b-quant.md` research in this repo). +- **Medium confidence:** exact GGUF file sizes for Qwen3-1.7B, Qwen3-0.6B, Llama-3.2-3B-Instruct, and + SmolLM3-3B — not individually re-verified against a specific quantizer's file tree since these were + eliminated on architectural (thinking-toggle) grounds before size became the deciding factor; treat + as typical/approximate, not exact. +- **Low confidence / not independently verified:** whether the current + `ghcr.io/ggml-org/llama.cpp:server-rocm` image (pulled fresh) still reproduces issue #20809's + misdetection — the issue is closed but no changelog/PR diff was fetched to confirm the underlying + detection logic was actually patched vs. the reporter simply adopting the `--reasoning off` + workaround. Recommend a live smoke test (send one tool-calling request, confirm the response lands in + `tool_calls` not `reasoning_content`, and time a trivial completion) before wiring this model in as + the production `fastModel`, the same caveat this repo's `qwen3.8-27b-tool-calling.md` already flags + for the 27B model. +- Whether Phi-4-mini-instruct has a comparably clean llama.cpp tool-calling track record was **not** + deep-dived (no issue-tracker search run against it) — it's flagged in §6 as a live alternative rather + than fully evaluated, since Qwen3-4B-Instruct-2507's architectural non-thinking guarantee and + same-family template consistency with the existing 27B deployment made it the clearer pick without + needing that extra research pass. diff --git a/scripts/update.sh b/scripts/update.sh index d1b051b..2ddd34f 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -88,6 +88,7 @@ docker compose build --pull echo "==> ensuring models are downloaded (skips already-present files)" docker compose --profile tools run --rm downloader +docker compose --profile tools run --rm downloader-fast echo "==> bringing up omniroute" docker compose up -d --wait omniroute -- 2.54.0