From 52a92f650896c66a51db05bc654ca92b309c1a72 Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Sun, 6 Sep 2026 22:06:59 +0200 Subject: [PATCH 1/3] fix: llama-server-fast context-size exhaustion breaking Auto Mode classifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real failure: "Auto Mode couldn't classify this action (Classifier stage 1 unavailable)". Reproduced directly against the server: {"error":{"message":"[400]: request (6186 tokens) exceeds the available context size (4096 tokens)"... LLAMA_FAST_CTX_SIZE=8192 is the TOTAL across every LLAMA_FAST_PARALLEL slot, not per-request — the main model's own .env.example comment already calls this out, missed it when llama-server-fast was set up (#44). With PARALLEL=2 that's 4096/slot, too small for a real classifier call (hints + environment + recent tool-call history). Fixed by dropping to a single slot (LLAMA_FAST_PARALLEL=1) rather than raising ctx-size — this service doesn't need concurrent classifier calls the way the main model needs concurrent chat sessions, so this costs no extra VRAM. The full 8192 now goes to the one slot. docker compose config -q validated. Refs #5 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01MrnMEdzeQzqZE5soVEXPCx --- .env.example | 19 +++++++++++++++---- docker-compose.yml | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 619dbc4..5108412 100644 --- a/.env.example +++ b/.env.example @@ -116,11 +116,22 @@ LLAMA_FAST_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf # Same reasoning as LLAMA_GPU_LAYERS above — full GPU offload, this model # is dense too. LLAMA_FAST_GPU_LAYERS=999 -# Classifier transcripts are truncated/bounded by qwen-code itself (see its -# own Auto Mode docs) — no need for anywhere near the 27B's huge context. -# 8192 keeps this instance's KV cache negligible. +# --ctx-size is the TOTAL across every LLAMA_FAST_PARALLEL slot, not per +# request — same halving already called out for the main model above. +# Was PARALLEL=2, silently halving this to 4096/slot — too small: a real +# classifier call (hints + environment + recent tool-call history) hit +# "exceeds the available context size (4096 tokens)" in practice, which +# qwen-code surfaces as "Auto Mode couldn't classify this action +# (Classifier stage 1 unavailable)" — see issue #5. Fixed by dropping to +# a single slot instead of raising ctx-size (no extra VRAM, and this +# service doesn't need concurrent classifier calls the way the main +# model needs concurrent chat sessions) — the full 8192 now goes to the +# one slot. If hints.allow/softDeny/hardDeny ever approach their +# 50-entries-each ceiling, raise LLAMA_FAST_CTX_SIZE instead — qwen-code +# caps those at 200 chars x 150 entries plus 40,000 chars of +# historical-action context, which can exceed 8192 tokens worst-case. LLAMA_FAST_CTX_SIZE=8192 -LLAMA_FAST_PARALLEL=2 +LLAMA_FAST_PARALLEL=1 # --- ComfyUI diffusion model (Qwen-Image, FP8 — see docs/research/ # image-generation-model-choice.md and issue #42) --- diff --git a/docker-compose.yml b/docker-compose.yml index 4e18455..5c0ed1d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,7 +70,7 @@ services: --port 8080 --n-gpu-layers ${LLAMA_FAST_GPU_LAYERS:-999} --ctx-size ${LLAMA_FAST_CTX_SIZE:-8192} - --parallel ${LLAMA_FAST_PARALLEL:-2} + --parallel ${LLAMA_FAST_PARALLEL:-1} --flash-attn on --cache-type-k q8_0 --cache-type-v q8_0 From 9def240a8ebdb1309bd8c30414e3082761b84f7c Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Sun, 6 Sep 2026 22:10:51 +0200 Subject: [PATCH 2/3] feat: update.sh force-syncs tracked config from .env.example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follows directly from the previous commit's caveat: this PR's own fix (LLAMA_FAST_PARALLEL=2 -> 1) wouldn't have taken effect on the server without a manual .env edit, because set_if_blank never touches an already-set value — by design, for secrets, but the same logic was silently protecting stale copies of ordinary tunable config too. Every KEY=VALUE line in .env.example with a real (non-blank) default is now force-synced into .env on every run. Secrets and host-resolved values are unaffected — .env.example already leaves those blank on purpose, so the sync loop naturally skips them and they keep going through set_if_blank as before. Trade-off, called out in both the script's header and the sync loop's own comment: there's no such thing as a persistent server-only override for these keys anymore — a hand-edited value not reflected in git gets reverted on the next run. That's the intended behavior. Verified against a simulated stale .env matching the real scenario from this PR: LLAMA_FAST_PARALLEL correctly overwritten 2 -> 1, an OMNIROUTE secret left untouched. bash -n and docker compose config -q both pass. Refs #5 --- scripts/update.sh | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/scripts/update.sh b/scripts/update.sh index 237be03..8141a9a 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -4,6 +4,13 @@ # exist, pulls, validates, rebuilds/re-pulls images, and recreates only what # changed — safe to run any time, including with nothing to do. # +# Tunable config values (LLAMA_*, ports, timeouts — anything with a real +# default in .env.example) are force-synced from .env.example every run, +# not just filled in when missing — see the sync loop below. Only actual +# secrets and host-resolved values (blank in .env.example) survive a +# server-side edit; there's no persistent server-only override for +# anything else. +# # omniroute's own routing/provider config (llama-server, search) lives in # its dashboard, not a checked-in file like the old litellm-config.yaml — # see issue #31 and docs/proxy-key-onboarding.md. @@ -27,6 +34,29 @@ git pull --ff-only [ -f .env ] || cp .env.example .env +echo "==> syncing tracked config values from .env.example" +# Every KEY=VALUE line in .env.example that has a real default (not +# blank) is ordinary tunable config, not a secret or host-specific +# value — .env.example itself already draws that line: secrets +# (OMNIROUTE_*_SECRET/_KEY/_SALT/_PASSWORD) and host-resolved values +# (SEARXNG_LAN_IP, COMFYUI_PUID/PGID, HOST_VIDEO_GID/RENDER_GID) are +# all left blank there on purpose, so this loop naturally skips them — +# they keep going through set_if_blank below instead. This force- +# overwrites .env with whatever's in git on every run, so a value +# hand-edited on the server (not in .env.example) gets silently +# reverted on the next update.sh — that's the point (git pull #48 bit +# us exactly because a stale .env value survived a code change and +# broke llama-server-fast, see #5/#49), but it means there's no such +# thing as a persistent server-only override for these keys anymore. +while IFS='=' read -r key value; do + [ -n "$value" ] || continue + if grep -qE "^${key}=" .env; then + sed -i "s|^${key}=.*|${key}=${value}|" .env + else + echo "${key}=${value}" >> .env + fi +done < <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=.+' .env.example) + # Handles all three cases: the KEY=value line is missing entirely (.env # predates that var being added to .env.example — sed can't fix what isn't # there, so this appends it), present but blank, or already set. From ea7b05fb999247c3bea42c0058b8dd59063c1573 Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Sun, 6 Sep 2026 22:20:27 +0200 Subject: [PATCH 3/3] feat: interactive per-key conflict resolution in update.sh's config sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the previous commit's blind force-overwrite with a real choice, per user feedback: force-overwriting server config without asking was the wrong default. - A tracked config value (real default in .env.example) that already matches .env is left alone silently — no prompt, no noise. - A value that DIFFERS is a conflict, shown on one screen (all conflicts together, not one prompt per key) via gum (charmbracelet/gum) — single static binary fetched as a release tarball into .cache/gum/ (gitignored), no build step, no package manager dependency. Falls back to a plain read-based prompt if gum can't be fetched (offline, unsupported arch). - Non-interactive (no TTY — cron, CI, piped): any conflict is a hard error (exit 1, lists every conflicting key) unless --force is passed, which accepts every new value automatically — matches how this PR's own fix needs to land unattended. - Secrets and host-resolved values are completely unaffected either way — untouched by this loop, same as before. Verified in an isolated sandbox against the exact scenario from this PR (stale LLAMA_FAST_PARALLEL=2 vs git's 1): - no TTY, no --force: exits 1, prints the diff, doesn't touch .env - no TTY, --force: LLAMA_FAST_PARALLEL corrected 2 -> 1, an OMNIROUTE secret confirmed untouched (not regenerated) docker compose config -q still passes. Not verified: the interactive gum path itself (needs a real TTY, couldn't allocate a pty in this sandbox) — worth confirming for real on the server, including that gum's release asset naming actually matches what ensure_gum() expects. Refs #5 --- .gitignore | 1 + scripts/update.sh | 132 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 112 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 74c31ad..77761ae 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ # to be committed to this repo. data/ .leankg/ +.cache/ diff --git a/scripts/update.sh b/scripts/update.sh index 8141a9a..d3c9054 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -5,11 +5,16 @@ # changed — safe to run any time, including with nothing to do. # # Tunable config values (LLAMA_*, ports, timeouts — anything with a real -# default in .env.example) are force-synced from .env.example every run, -# not just filled in when missing — see the sync loop below. Only actual -# secrets and host-resolved values (blank in .env.example) survive a -# server-side edit; there's no persistent server-only override for -# anything else. +# default in .env.example) are synced from .env.example every run. A value +# already matching is left alone silently. A value that DIFFERS from the +# server's current .env is a conflict: interactively, you're shown every +# conflict on one screen (via gum) and choose which to accept — unpicked +# keys keep the server's current value. Non-interactively (no TTY — cron, +# CI, piped), any conflict is a hard error unless --force is passed, which +# accepts every new value automatically. Secrets and host-resolved values +# (blank in .env.example — OMNIROUTE_*_SECRET/_KEY/_SALT/_PASSWORD, +# SEARXNG_LAN_IP, COMFYUI_PUID/PGID, HOST_VIDEO_GID/RENDER_GID) are never +# touched by this — they keep going through set_if_blank as before. # # omniroute's own routing/provider config (llama-server, search) lives in # its dashboard, not a checked-in file like the old litellm-config.yaml — @@ -21,6 +26,14 @@ set -euo pipefail cd "$(dirname "$0")/.." +FORCE=false +for arg in "$@"; do + case "$arg" in + --force) FORCE=true ;; + *) echo "Usage: $0 [--force]" >&2; exit 1 ;; + esac +done + # Must run before anything else touches a file this script itself reads # (docker-compose.yml, .env.example, this script's own remaining lines) — # a self-updating script isn't guaranteed atomic against its own file @@ -35,28 +48,105 @@ git pull --ff-only [ -f .env ] || cp .env.example .env echo "==> syncing tracked config values from .env.example" -# Every KEY=VALUE line in .env.example that has a real default (not -# blank) is ordinary tunable config, not a secret or host-specific -# value — .env.example itself already draws that line: secrets -# (OMNIROUTE_*_SECRET/_KEY/_SALT/_PASSWORD) and host-resolved values -# (SEARXNG_LAN_IP, COMFYUI_PUID/PGID, HOST_VIDEO_GID/RENDER_GID) are -# all left blank there on purpose, so this loop naturally skips them — -# they keep going through set_if_blank below instead. This force- -# overwrites .env with whatever's in git on every run, so a value -# hand-edited on the server (not in .env.example) gets silently -# reverted on the next update.sh — that's the point (git pull #48 bit -# us exactly because a stale .env value survived a code change and -# broke llama-server-fast, see #5/#49), but it means there's no such -# thing as a persistent server-only override for these keys anymore. +# ponytail: gum (charmbracelet/gum) is a single static binary, fetched as a +# release tarball — no build step, no package-manager dependency. Cached +# under .cache/gum/ (gitignored) so repeat runs don't re-download. Release +# asset naming (gum__Linux_.tar.gz) follows charm's standard +# goreleaser convention but hasn't been exercised against a real download +# on this exact host yet — if it 404s, check +# https://github.com/charmbracelet/gum/releases for the current naming. +GUM_VERSION="0.14.5" +GUM_DIR="$(pwd)/.cache/gum" +GUM_BIN="$GUM_DIR/gum" +ensure_gum() { + command -v gum >/dev/null 2>&1 && { echo "gum"; return; } + [ -x "$GUM_BIN" ] && { echo "$GUM_BIN"; return; } + mkdir -p "$GUM_DIR" + local arch tmpdir url + case "$(uname -m)" in + x86_64) arch="x86_64" ;; + aarch64|arm64) arch="arm64" ;; + *) echo "no gum build for $(uname -m), falling back to plain prompts" >&2; echo ""; return ;; + esac + url="https://github.com/charmbracelet/gum/releases/download/v${GUM_VERSION}/gum_${GUM_VERSION}_Linux_${arch}.tar.gz" + tmpdir="$(mktemp -d)" + if curl -fsSL "$url" | tar -xz -C "$tmpdir" 2>/dev/null; then + find "$tmpdir" -name gum -type f -exec cp {} "$GUM_BIN" \; + chmod +x "$GUM_BIN" 2>/dev/null || true + fi + rm -rf "$tmpdir" + [ -x "$GUM_BIN" ] && echo "$GUM_BIN" || echo "" +} + +# Collect every key where .env.example has a real (non-blank) default: +# missing from .env -> just add it (no conflict, nothing to decide); +# present and identical -> leave alone silently; present and different -> +# a conflict to resolve below. +conflict_keys=() +conflict_old=() +conflict_new=() while IFS='=' read -r key value; do [ -n "$value" ] || continue - if grep -qE "^${key}=" .env; then - sed -i "s|^${key}=.*|${key}=${value}|" .env - else + current="$(grep -E "^${key}=" .env | head -1 | cut -d= -f2-)" + if ! grep -qE "^${key}=" .env; then echo "${key}=${value}" >> .env + elif [ "$current" != "$value" ]; then + conflict_keys+=("$key") + conflict_old+=("$current") + conflict_new+=("$value") fi done < <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=.+' .env.example) +if [ "${#conflict_keys[@]}" -gt 0 ]; then + if [ "$FORCE" = true ]; then + for i in "${!conflict_keys[@]}"; do + key="${conflict_keys[$i]}"; new="${conflict_new[$i]}" + sed -i "s|^${key}=.*|${key}=${new}|" .env + echo "${key}: ${conflict_old[$i]} -> ${new} (--force)" + done + elif [ ! -t 0 ] || [ ! -t 1 ]; then + echo "ERROR: ${#conflict_keys[@]} config value(s) in .env differ from .env.example, and this isn't an interactive terminal:" >&2 + for i in "${!conflict_keys[@]}"; do + echo " ${conflict_keys[$i]}: ${conflict_old[$i]} (current) vs ${conflict_new[$i]} (.env.example)" >&2 + done + echo "Re-run interactively to choose per-key, or pass --force to accept every new value." >&2 + exit 1 + else + gum_bin="$(ensure_gum)" + labels=() + for i in "${!conflict_keys[@]}"; do + labels+=("${conflict_keys[$i]}: ${conflict_old[$i]} -> ${conflict_new[$i]}") + done + if [ -n "$gum_bin" ]; then + selected="$(printf '%s\n' "${labels[@]}" | "$gum_bin" choose --no-limit --selected "$(printf '%s\n' "${labels[@]}" | paste -sd,)" --header "Config differs from .env.example — selected keys take the new value, unselected keep the server's current value:")" + else + # ponytail: plain-bash fallback if gum couldn't be fetched (offline, + # unsupported arch) — same one-screen-of-conflicts idea, cruder UI. + echo "Config differs from .env.example. Enter space-separated numbers to KEEP the server's current value (all others take the new value), or press enter to take every new value:" + for i in "${!conflict_keys[@]}"; do + echo " $((i+1))) ${labels[$i]}" + done + read -r -p "> " keep_nums + selected="" + for i in "${!conflict_keys[@]}"; do + case " $keep_nums " in + *" $((i+1)) "*) ;; + *) selected="${selected}${labels[$i]}"$'\n' ;; + esac + done + fi + for i in "${!conflict_keys[@]}"; do + key="${conflict_keys[$i]}"; new="${conflict_new[$i]}" + if printf '%s\n' "$selected" | grep -qxF "${labels[$i]}"; then + sed -i "s|^${key}=.*|${key}=${new}|" .env + echo "${key}: ${conflict_old[$i]} -> ${new}" + else + echo "${key}: kept ${conflict_old[$i]} (server value)" + fi + done + fi +fi + # Handles all three cases: the KEY=value line is missing entirely (.env # predates that var being added to .env.example — sed can't fix what isn't # there, so this appends it), present but blank, or already set.