From 633292b291476402f3484e14e7d354c26c9fb83a Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Sat, 5 Sep 2026 11:29:12 +0200 Subject: [PATCH] fix(omniroute): raise stream-idle timeout to 180s Contended prefill (see LLAMA_PARALLEL) can outrun the ~95s dashboard value before first token, causing OmniRoute to cancel still-working requests. Track it in git via STREAM_IDLE_TIMEOUT_MS instead of a dashboard-only setting. Co-Authored-By: Claude Sonnet 5 --- .env.example | 4 ++++ docker-compose.yml | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/.env.example b/.env.example index 3233082..3d6e884 100644 --- a/.env.example +++ b/.env.example @@ -61,6 +61,10 @@ SEARXNG_LAN_IP= # published at all — see docker-compose.yml's omniroute service comment. OMNIROUTE_API_PORT=20129 OMNIROUTE_DASHBOARD_PORT=20128 +# SSE inactivity timeout before OmniRoute gives up on a streaming request and +# cancels it (which cancels the matching llama-server task too). 180s gives +# contended prefill (see LLAMA_PARALLEL above) room to produce a first token. +OMNIROUTE_STREAM_IDLE_TIMEOUT_MS=180000 # Random values, filled in automatically by ./scripts/update.sh — leave # blank. Bootstrap dashboard admin password (log in at the dashboard port, # change it there afterwards — this is only the first-boot value): diff --git a/docker-compose.yml b/docker-compose.yml index 59ce820..a925ad4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,6 +92,14 @@ services: # Docker guide — every client here is a coding CLI, which needs the # larger figure the guide recommends. Paired with mem_limit below. - OMNIROUTE_MEMORY_MB=8192 + # Default 300000 (5 min) per OmniRoute's own docs, but this deployment + # had it dialed down elsewhere (dashboard) to ~95s — too tight for a + # contended local llama-server: large-context prefill under multiple + # concurrent slots can outrun that before the first SSE token arrives, + # so OmniRoute cancels a request that was actually still working (see + # LLAMA_PARALLEL above for the other half of this fix). Raised here so + # it's tracked in git instead of a dashboard-only setting. + - STREAM_IDLE_TIMEOUT_MS=${OMNIROUTE_STREAM_IDLE_TIMEOUT_MS:-180000} # Same reasoning as litellm's extra_hosts entry below — ai-stack's bridge # network can't resolve search.home on its own. extra_hosts: