From fb7cfc914853ee228374d28bec7ac45c82a48fbc Mon Sep 17 00:00:00 2001 From: ArthurErlich Date: Wed, 2 Sep 2026 22:59:53 +0200 Subject: [PATCH] fix(litellm): raise qwen3.8-27b-local max_tokens floor 4096 -> 16384 Confirmed in llama-server logs: a real request hit exactly n_gen=4096 (the old floor) and returned no answer -- reasoning_content alone ate the whole budget before any content was written, exactly the failure mode this config's own comment predicted. 16384 is the user-chosen middle ground between 8192 and 32768: ~10 min worst-case at ~26.7 t/s, well under the 65536-token context window. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_018WHfjWrSEcGhCoeu6dQfDa --- litellm-config.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm-config.yaml b/litellm-config.yaml index fcbb3c2..1f105c5 100644 --- a/litellm-config.yaml +++ b/litellm-config.yaml @@ -13,7 +13,12 @@ model_list: # llama.cpp's low default, so the model runs out mid-thought and # content comes back empty. This is a floor, not a cap — any caller # that passes its own max_tokens still overrides it. - max_tokens: 4096 + # Raised from 4096: confirmed in the wild (llama-server logs) that + # 4096 wasn't enough — reasoning_content alone ate the whole budget on + # a real request (n_gen = 4096 exactly, no answer ever written). At + # ~26.7 t/s and a 65536-token context window, 16384 is a ~10-minute + # worst case, not the full ~20-minute worst case 32768 would be. + max_tokens: 16384 model_info: # Shadow cloud-cost estimate — priced against Claude Sonnet 5's published # rate, not real spend (this proxy only ever routes to the local model).