Merge pull request 'docs(research): add qwen3.8-27b decode speed analysis' (#68) from docs/decode-speed-research into main

Reviewed-on: #68

.env.example

@@ -1,138 +1,37 @@
-# Copy to .env and adjust — or just run ./scripts/update.sh, which creates
+# Template for .env, managed by ./scripts/update.sh — see docs/operations.md#env-sync
-# .env from this file and fills in every secret/key below it can generate
-# itself (see each var's comment). All values below are defaults baked into
-# docker-compose.yml — only uncomment/change what you actually want to
-# override.
 
-# --- llama.cpp / model ---
+# --- llama.cpp / models --- see docs/architecture.md#llama-server
 LLAMA_MODEL_FILE=Qwen3.8-27B-UD-Q4_K_XL.gguf
-# 999 = every layer on GPU (this model is dense, not MoE, and already fits
-# fully in 32GB VRAM — see docs/research/qwen3.8-27b-quant.md). Lower this
-# to leave that many fewer layers on GPU and push the rest to CPU/system RAM
-# if something else is contending for VRAM — llama.cpp has no separate
-# "RAM offload" flag, --n-gpu-layers *is* the RAM-offload knob for a dense
-# model. Don't reach for --n-cpu-moe/--cpu-moe/--override-tensor "exps" —
-# those target Mixture-of-Experts models (e.g. Qwen3.8-2.4T-A95B), not this
-# one, and are no-ops here.
-# There's no separate "then SSD" tier to enable either: llama.cpp mmaps the
-# model file by default (no --no-mmap here), so if GPU+RAM ever can't hold
-# the working set, the OS pages the rest in from disk automatically — an
-# implicit, slow last resort, not a config knob. An explicit tiered SSD
-# offload has been an open llama.cpp feature request since 2025 (still
-# unimplemented): https://github.com/ggml-org/llama.cpp/discussions/12507
 LLAMA_GPU_LAYERS=999
-# 262144 = this model's true max (max_position_embeddings in Qwen/Qwen3.8-27B's
-# config.json) — the largest --ctx-size llama.cpp will even accept for it.
-# fp16 KV cache at full context would be ~16GB, on top of 17.6GB weights =
-# ~33.6GB, which does NOT fit the 32GB R9700 on its own. docker-compose.yml
-# now runs --cache-type-k/v q8_0, which roughly halves KV memory (~8GB at
-# this size) — total ~25.6GB, ~6GB headroom, the same footprint the old
-# 131072 fp16 setting used. See docs/research/qwen3.8-27b-quant.md.
 LLAMA_CTX_SIZE=262144
-# Concurrent request slots — the real hardware ceiling for this GPU, not a
-# tunable to raise for throughput (was implicitly 4, llama.cpp's compiled-in
-# default; dropped to 2 because more contended prefill was blowing requests
-# past OmniRoute's idle timeout — see OMNIROUTE_STREAM_IDLE_TIMEOUT_MS below).
-# The 3rd+ request now queues on llama.cpp itself instead — its own queue has
-# no timeout (tools/server/server-queue.cpp), it just waits for a slot — so
-# the timeout that matters moved to OmniRoute's per-connection
-# providerSpecificData.timeoutMs (dashboard/API only, not in this file; see
-# handoff notes in the issue tracker). Each slot gets LLAMA_CTX_SIZE /
-# LLAMA_PARALLEL tokens of context — real sessions have hit ~66K tokens, so
-# don't drop LLAMA_CTX_SIZE without checking that per-slot number stays
-# comfortably above observed usage.
 LLAMA_PARALLEL=2
+LLAMA_CACHE_RAM=8192
+LLAMA_KV_CACHE_TYPE=q8_0
 
-# Dedicated GPU-resident backend for qwen-code's tool-call harmfulness
+# Blank = speculative decoding off; see docs/architecture.md#llama-server
-# classifier (fastModel in ~/.qwen/settings.json) — see docker-compose.yml's
+LLAMA_SPEC_ARGS=--spec-type draft-dflash -md /models/Qwen3.8-27B-DFlash2-Q8_0.gguf --spec-draft-n-max 3 --spec-draft-p-min 0.5
-# qwen-classifier service comment for the why and the VRAM/context math.
-LLAMA_CLASSIFIER_MODEL_FILE=Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf
 
-# --- Lazytainer ---
+# --- SearXNG web search --- see docs/gateway.md#web-search
-# Seconds of inactivity before llama-server is stopped. 900 = 15 min.
+# Auto-filled by update.sh — leave blank.
-LAZYTAINER_INACTIVE_TIMEOUT=900
-
-# --- SearXNG web search (see docs/research/litellm-searxng-search.md) ---
-# Resolved automatically by ./scripts/update.sh from search.home on this
-# host — leave blank. Only set by hand if that resolution fails (e.g.
-# search.home isn't a static DHCP reservation and its IP drifted).
 SEARXNG_LAN_IP=
 
-# --- OmniRoute gateway (see docs/proxy-key-onboarding.md, docs/network-access.md) ---
+# --- OmniRoute gateway --- see docs/network-access.md#gateway
-# OMNIROUTE_PORT is the host-published port (reverse-proxied by NPM) — kept
-# at 4000, same as the old LiteLLM setup, so existing NPM/firewall config
-# doesn't need to change. It's mapped via plain Docker port publishing onto
-# API_PORT, omniroute's own container-internal port (left at its default,
-# not reconfigured to match). The dashboard (DASHBOARD_PORT) is never
-# published at all — see docker-compose.yml's omniroute service comment.
 OMNIROUTE_API_PORT=20129
 OMNIROUTE_DASHBOARD_PORT=20128
-# SSE inactivity timeout before OmniRoute gives up on a streaming request and
+# see docs/gateway.md#timeouts
-# cancels it (which cancels the matching llama-server task too). 180s gives
-# contended prefill (see LLAMA_PARALLEL above) room to produce a first token.
 OMNIROUTE_STREAM_IDLE_TIMEOUT_MS=180000
-# Wait budget for the *first* SSE token specifically (distinct from the
-# inter-chunk timeout above) — see
-# docs/research/omniroute-non-ping-sse-stream-timeout.md. 30 min covers a
-# contended, large-context prefill even after retries eat into the budget.
 OMNIROUTE_REQUEST_TIMEOUT_MS=1800000
-# Random values, filled in automatically by ./scripts/update.sh — leave
+# Secrets: auto-filled by update.sh — leave blank. See docs/operations.md#secrets
-# blank. Bootstrap dashboard admin password (log in at the dashboard port,
-# change it there afterwards — this is only the first-boot value):
 OMNIROUTE_INITIAL_PASSWORD=
-# Signs dashboard session cookies:
 OMNIROUTE_JWT_SECRET=
-# Encrypts API key values at rest in omniroute's SQLite DB:
 OMNIROUTE_API_KEY_SECRET=
-# Encrypts the whole SQLite DB at rest. Do not change after first run —
+# NEVER change after first run — OmniRoute's existing data becomes unreadable.
-# existing encrypted data becomes unreadable if you do (same caveat as
-# LiteLLM's old LITELLM_SALT_KEY):
 OMNIROUTE_STORAGE_ENCRYPTION_KEY=
-# Per-deployment salts — random is fine, just needs to be stable:
 OMNIROUTE_MACHINE_ID_SALT=
 OMNIROUTE_CLI_SALT=
-# Required (production) — shared secret for the internal Codex Responses
-# WebSocket bridge. Random value, filled in automatically:
 OMNIROUTE_WS_BRIDGE_SECRET=
-# Per-workload virtual keys (one per client that calls the gateway) have no
-# scripted /key/generate equivalent yet — omniroute's key-creation endpoint
-# needs a dashboard login session, not a static bearer key (see issue #37).
-# Mint them by hand in the dashboard, add a KEY=value line here per workload
-# as you onboard one. See docs/proxy-key-onboarding.md.
 
-# --- ComfyUI (local image generation, see issue #38 wayfinder map) ---
+# --- GPU access --- see docs/architecture.md#gpu-access
-# yurisasc/comfyui-rocm7.1 manages GPU-group access via these GID/UID env
+# Auto-filled by update.sh — leave blank.
-# vars rather than relying solely on docker-compose.yml's group_add.
-# Resolved automatically from the host by ./scripts/update.sh — leave blank.
-COMFYUI_PUID=
-COMFYUI_PGID=
-
-# Shared by every GPU-touching service (llama-server, llama-server-fast,
-# comfyui) for group_add: — resolved to real host GIDs by ./scripts/update.sh
-# rather than left as plain group names in docker-compose.yml, because Docker
-# resolves a *named* group_add entry against the container's own /etc/group,
-# not the host's, and fails unpredictably when the image doesn't define one
-# (worse with multiple GPU services racing on the same lookup at once — see
-# docs/research/rocm-gpu-pin-and-render-group.md and issue #5). Leave blank.
 HOST_VIDEO_GID=
 HOST_RENDER_GID=
-
-# --- ComfyUI diffusion model (Qwen-Image 2.1, uncensored GGUF — see
-# docs/research/image-generation-model-choice.md and issue #42) ---
-# Three files, all from abenzerps/Qwen-Image-2.1-Uncensored-GGUF on
-# HuggingFace: GGUF diffusion weights, safetensors text encoder, safetensors
-# VAE. Q4_K_M is the repo's own recommended size/quality balance (~4.6GB);
-# the int8 text encoder trades some quality for ~8GB less RAM than bf16.
-#
-# Requires the ComfyUI-GGUF custom node (github.com/city96/ComfyUI-GGUF) —
-# NOT confirmed bundled in yurisasc/comfyui-rocm7.1. If the comfyui
-# container doesn't show a "Unet Loader (GGUF)" node, `git clone` it into
-# the image's custom_nodes dir and reinstall its requirements.
-COMFYUI_DIFFUSION_MODEL_FILE=qwen-image-2.1-UC-Q4_K_M.gguf
-COMFYUI_TEXT_ENCODER_FILE=qwen3vl_8b_int8_convrot.safetensors
-COMFYUI_VAE_FILE=qwen_image_2.1_vae_bf16.safetensors
-
-# --- RAG databases (qdrant + neo4j, see wayfinder notes) ---
-# No auth on qdrant (its default) — same trust boundary as llama-server:
-# ai-stack is not exposed off-box. Random, filled in automatically:
-NEO4J_PASSWORD=

.gitignore

@@ -6,3 +6,5 @@ data/
 .leankg/
 .cache/
 .qwen/tmp
+# speed-bench output, see docs/operations.md#speed-bench
+bench-results/

.serena/.gitignore

@@ -0,0 +1,2 @@
+/cache
+/project.local.yml

.serena/project.yml

@@ -0,0 +1,187 @@
+# the name by which the project can be referenced within Serena/when chatting with the LLM.
+project_name: "LLM-Server"
+
+# list of language servers to start when using the LSP backend; choose from:
+#   ada                    al                     angular                ansible                astro
+#   bash                   bsl                    clojure                cpp                    cpp_ccls
+#   crystal                csharp                 csharp_omnisharp       cue                    dart
+#   deno                   elixir                 elm                    erlang                 fortran
+#   fsharp                 gdscript               gleam                  go                     groovy
+#   haskell                haxe                   hlsl                   html                   java
+#   json                   julia                  julia_fatou            kotlin                 latex
+#   lean4                  lua                    lua_emmylua            luau                   markdown
+#   matlab                 msl                    nextflow               nix                    ocaml
+#   pascal                 perl                   php                    php_devsense           php_phpactor
+#   php_phpantom           powershell             python                 python_basedpyright    python_jedi
+#   python_pyrefly         python_ty              qml                    r                      rego
+#   ruby                   ruby_solargraph        rust                   scala                  scss
+#   solidity               svelte                 swift                  systemverilog          terraform
+#   toml                   typescript             typescript_tsgo        typescript_vts         vue
+#   vbnet                  wolfram                yaml                   zig
+#   (This list may be outdated; generated with scripts/print_language_list.py;
+#   For the current list, see values of the LanguageServerId enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
+# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Astro projects, use astro (subsumes typescript/javascript for .astro projects; requires npm)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+#   - External Python adapters may add further registered IDs; install the adapter package first
+#     and then use its ID here, for example: example
+# Special requirements:
+#   Some language servers require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple language servers, the first language server that supports a given file will be used for that file.
+# The first language server is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+language_servers:
+- bash
+
+# the encoding used by text files in the project
+# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
+encoding: "utf-8"
+
+# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
+# the command runs in the project root directory and is only executed if the project is trusted
+# (see trusted_project_path_patterns in the global configuration).
+# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
+# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
+# backstop for non-terminating commands; on expiry the process is killed and activation continues.
+# example: activation_command: "npx nx run-many -t build"
+activation_command:
+
+# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
+# must be a positive number.
+activation_command_timeout: 180.0
+
+# line ending convention to use when writing source files.
+# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
+# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
+line_ending:
+
+# The language backend to use for this project.
+# If not set, the global setting from serena_config.yml is used.
+# Valid values: LSP, JetBrains
+# Note: the backend is fixed at startup. If a project with a different backend
+# is activated post-init, an error will be returned.
+language_backend:
+
+# The interface through which the agent (LLM) accesses Serena's functionality (overrides the global setting).
+# Valid values: tools, REPL (see the global configuration for details); leave empty to use the global setting.
+# Note: the interface is fixed at startup. If a project is activated post-init, its setting is not applied.
+agent_interface:
+
+# whether to use project's .gitignore files to ignore files
+ignore_all_files_in_gitignore: true
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
+ls_specific_settings: {}
+
+# list of workspace folder paths (LSP backend only).
+# These folders will be used to build up Serena's symbol index.
+# Paths must be within the project root and should thus be relative to the project root.
+# Furthermore, the paths should not be filtered by ignore settings.
+# Default setting: The entire project root folder (".") is considered.
+# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
+#   ls_workspace_folders:
+#     - "./subproject1"
+#     - "./subproject2"
+ls_workspace_folders:
+- "."
+
+# list of additional workspace folder paths for cross-package reference support.
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries, but these folders are not indexed by Serena,
+# i.e. the respective symbols will not be found using Serena's symbol search tools.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+ls_additional_workspace_folders: []
+
+# list of additional paths to ignore in this project.
+# Same syntax as gitignore, so you can use * and **.
+# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
+# Example:
+#   ignored_paths:
+#     - "examples/**"
+#     - ".worktrees/**"
+#     - "**/bin/**"
+#     - "**/obj/**"
+# Note: global ignored_paths from serena_config.yml are also applied additively.
+ignored_paths: []
+
+# whether the project is in read-only mode
+# If set to true, all editing tools will be disabled and attempts to use them will result in an error
+# Added on 2025-04-18
+read_only: false
+
+# list of tool names to exclude.
+# This extends the existing exclusions (e.g. from the global configuration)
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+excluded_tools: []
+
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+fixed_tools: []
+
+# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.find_symbol") to exclude from the REPL.
+# This extends the existing exclusions (e.g. from the global configuration).
+excluded_apis: []
+
+# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.get_diagnostics_for_symbol") to include in the REPL
+# that would otherwise be disabled (particularly optional methods, which are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+included_apis: []
+
+# list of mode names that are to be activated by default, overriding the setting in the global configuration.
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
+# for this project.
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+default_modes:
+
+# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+added_modes:
+
+# initial prompt for the project. It will always be given to the LLM upon activating the project
+# (contrary to the memories, which are loaded on demand).
+initial_prompt: ""
+
+# time budget (seconds) per tool call for the retrieval of additional symbol information
+# such as docstrings or parameter information.
+# This overrides the corresponding setting in the global configuration; see the documentation there.
+# If null or missing, use the setting from the global configuration.
+symbol_info_budget:
+
+# list of regex patterns which, when matched, mark a memory entry as read‑only.
+# Extends the list from the global configuration, merging the two lists.
+read_only_memory_patterns: []
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []

CLAUDE.md

@@ -8,6 +8,10 @@ Issues live as Gitea issues on `git.arthurerlich.de` (repo `haylan/LLM-Server`);
 
 Single-context: `CONTEXT.md` + `docs/adr/` at the repo root. See `docs/agents/domain.md`.
 
+### Writing docs
+
+Before editing `README.md`, `docs/`, or a comment in `docker-compose.yml` / `.env.example` / `scripts/`, read `docs/agents/writing-docs.md`.
+
 ### Deploying changes
 
 The running stack lives on a separate box (the R9700 server), not wherever this repo is being edited. After **any** change to `docker-compose.yml`, `.env.example`, or a script under `scripts/`, commit/push it, then run `./scripts/update.sh` on the server to apply it — don't just describe the change as done. If this session doesn't have shell access to the server, say so explicitly and tell the user to run it themselves rather than leaving it unsaid.

CONTEXT.md

@@ -0,0 +1,59 @@
+# LLM-Server
+
+A self-hosted inference stack: local models served on one GPU machine, reached by other tools through a single authenticated entry point. This file is a glossary only — for how things are configured, follow the links in the docs to the source.
+
+## Language
+
+### Machine
+
+**Server**:
+The one machine that runs the stack.
+_Avoid_: AI box, R9700 box, this machine
+
+**Host**:
+The Docker host, as opposed to a container — only used for that distinction.
+_Avoid_: using it as a synonym for Server
+
+### Serving models
+
+**Backend**:
+A model server process that serves one model to the gateway (`llama-server`).
+_Avoid_: provider, model server
+
+**Slot**:
+One concurrent request lane inside a backend; a backend handles as many requests at once as it has slots.
+_Avoid_: parallel request, lane
+
+**Main model**:
+The large model that does the actual work for workloads.
+_Avoid_: chat model
+
+**Draft model**:
+The small speculative-decoding drafter that runs beside the main model inside `llama-server` (DFlash2).
+_Avoid_: classifier model (the separate classifier backend was removed)
+
+### Reaching the models
+
+**Gateway**:
+The single entry point every model call goes through (OmniRoute).
+_Avoid_: proxy, AI proxy
+
+**Provider connection**:
+The gateway's registration of one backend.
+_Avoid_: provider (alone), connection (alone)
+
+**API key**:
+A credential issued by the gateway to exactly one workload.
+_Avoid_: virtual key, workload key
+
+**Workload**:
+Anything that calls the gateway with its own API key.
+_Avoid_: client, consumer
+
+**Coding CLI**:
+A workload that is an interactive coding agent (Claude Code, Kimi CLI, OpenCode, Qwen Code).
+_Avoid_: harness
+
+**Reverse proxy**:
+The existing Nginx Proxy Manager instance that publishes the gateway under hostnames; not part of this stack.
+_Avoid_: proxy (alone)

README.md

@@ -1,35 +1,29 @@
 # LLM-Server
 
-Local AI inference stack: llama.cpp (ROCm) serving Qwen3.8-27B on an AMD Radeon AI PRO R9700, fronted by the OmniRoute AI gateway, with Lazytainer auto-suspending the inference container when idle.
+> **Source of truth:** [`docker-compose.yml`](docker-compose.yml) (what runs) and [`scripts/update.sh`](scripts/update.sh) (how it's deployed). The docs explain *why*; if they disagree with the source, the source wins.
 
-See the wayfinder map ([issue #1](https://git.arthurerlich.de/haylan/LLM-Server/issues/1)) for the full architecture rationale and open questions.
+A self-hosted inference stack on one AMD GPU server. A llama.cpp **backend** serving the main model sits behind an OmniRoute **gateway**. Every **workload**, mostly coding CLIs, reaches it through the gateway with its own **API key**. Terms are defined in [`CONTEXT.md`](CONTEXT.md).
 
 ## Quickstart
 
+On the server:
+
 ```bash
 ./scripts/update.sh
 ```
 
-`update.sh` creates `.env` from `.env.example` if missing, fills in every random secret it can generate itself (via `openssl`, `SEARXNG_LAN_IP` resolved from `search.home` on this host), downloads the model GGUF into the `models` volume if it's not there yet, then pulls/builds/brings up the whole stack. Safe to re-run any time — it only fills in what's still blank, skips the model if already downloaded, and only recreates what changed.
+It creates and syncs `.env`, generates secrets, downloads the models, and brings the stack up. It's safe to re-run at any time. What it does and why: [`docs/operations.md`](docs/operations.md).
 
-llama.cpp's own API is internal-only — everything routes through the AI gateway below.
+## Docs
 
-Pointing Claude Code CLI, Kimi CLI, OpenCode CLI, or Qwen Code CLI at the local endpoint: see [`docs/coding-cli-setup/`](docs/coding-cli-setup/index.md).
+| Doc | What it covers |
-
+|---|---|
-**Known risk**: Qwen3.8-27B's tool-calling reliability against llama.cpp's Anthropic shim is not yet verified (open upstream parser bugs against its model lineage) — see `docs/research/qwen3.8-27b-tool-calling.md`.
+| [`CONTEXT.md`](CONTEXT.md) | Glossary: the terms every doc uses |
-
+| [`docs/architecture.md`](docs/architecture.md) | Each service, why it exists, and why it's configured the way it is |
-## AI gateway (OmniRoute)
+| [`docs/gateway.md`](docs/gateway.md) | OmniRoute: dashboard, provider connections, API keys, timeouts, web search |
-
+| [`docs/network-access.md`](docs/network-access.md) | Who can reach which port, and why |
-An [AI gateway/proxy](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) fronts llama.cpp: per-workload API keys and usage tracking. As of [issue #31](https://git.arthurerlich.de/haylan/LLM-Server/issues/31) this is [OmniRoute](https://github.com/diegosouzapw/OmniRoute), replacing the original LiteLLM setup. `./scripts/update.sh` handles most of OmniRoute's secrets (see `.env.example`); per-workload API keys still need minting by hand in the dashboard — see [`docs/proxy-key-onboarding.md`](docs/proxy-key-onboarding.md).
+| [`docs/operations.md`](docs/operations.md) | What `update.sh` does: env sync, secrets, downloads; speed bench |
-
+| [`docs/coding-cli-setup/`](docs/coding-cli-setup/index.md) | Pointing Claude Code, Kimi CLI, OpenCode and Qwen Code at the gateway |
-- Gateway API: `http://<this-machine>:${OMNIROUTE_PORT:-4000}/v1` locally, or `proxy-ai.home` / `proxy-ai.haylan.ch` once routed through NPM — see [`docs/network-access.md`](docs/network-access.md).
+| [`docs/research/`](docs/research/README.md) | Dated research and decision records, each marked current or superseded |
-- Dashboard (key/provider management): LAN/host-only, never published to the internet — see `docs/network-access.md`.
+| [`docs/server-planing.md`](docs/server-planing.md) | Background research: a 4× GPU rack build. Not an active plan |
-- Issuing a key for a new workload: [`docs/proxy-key-onboarding.md`](docs/proxy-key-onboarding.md).
+| [`docs/agents/`](docs/agents/) | Instructions for coding agents: [issue tracker](docs/agents/issue-tracker.md), [domain docs](docs/agents/domain.md), [writing docs](docs/agents/writing-docs.md) |
-
-Coding CLIs (see [`docs/coding-cli-setup/`](docs/coding-cli-setup/index.md)) route through the gateway — llama-server has no published host port. **Not yet verified**: none of this has been smoke-tested on real hardware yet — see [issue #31](https://git.arthurerlich.de/haylan/LLM-Server/issues/31)'s tickets for the open items (provider registration, per-workload key minting).
-
-**Note on this choice**: OmniRoute's own docs (`docs/security/STEALTH_GUIDE.md`, `MITM-TPROXY-DECRYPT.md`, `PUBLIC_CREDS.md` in its repo) describe shipped features for evading AI-provider client detection, system-wide HTTPS interception via a locally-installed root CA, and hiding credentials from secret scanners. None of that is used by this stack's configuration, but it's a real characteristic of the upstream project — see issue #31's Notes for the full research trail before extending this integration further.
-
-### Web search
-
-The gateway also fronts SearXNG-backed web search — see `docs/research/litellm-searxng-search.md` for the original research (still applicable — same standalone-endpoint pattern, see issue #31's #35).

docker-compose.yml

@@ -5,17 +5,13 @@ services:
     devices:
       - /dev/kfd
       - /dev/dri
-    # Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in
+    # see docs/architecture.md#gpu-access (group_add GIDs, GPU_MAX_HW_QUEUES)
-    # .env.example and docs/research/rocm-gpu-pin-and-render-group.md.
     group_add:
       - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
       - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
     security_opt:
       - seccomp=unconfined
     ipc: host
-    # Caps this process's HIP hardware-queue allocation — works around
-    # ROCm/ROCm#5706 (GPU pinned at 100%/boost-clock whenever two
-    # concurrent HIP contexts touch this card). See the research doc above.
     environment:
       - GPU_MAX_HW_QUEUES=1
     volumes:
@@ -27,245 +23,50 @@ services:
       --n-gpu-layers ${LLAMA_GPU_LAYERS:-999}
       --ctx-size ${LLAMA_CTX_SIZE:-262144}
       --parallel ${LLAMA_PARALLEL:-2}
+      --kv-unified
       --flash-attn on
-      --cache-type-k q8_0
+      --cache-type-k ${LLAMA_KV_CACHE_TYPE:-q8_0}
-      --cache-type-v q8_0
+      --cache-type-v ${LLAMA_KV_CACHE_TYPE:-q8_0}
-      --cache-reuse 256
+      --cache-ram ${LLAMA_CACHE_RAM:-8192}
-      --jinja
+      --no-cache-idle-slots
-    # --cache-reuse 256: reuse cached KV for any matching prompt chunk of at
+      ${LLAMA_SPEC_ARGS:-}
-    # least 256 tokens (KV-shift, no reprocessing) instead of reprefilling
-    # from scratch every request. Directly targets the actual root cause
-    # behind the OmniRoute non-ping SSE timeout, not just the symptom — see
-    # docs/research/omniroute-non-ping-sse-stream-timeout.md. Pairs with
-    # OmniRoute's promptCacheAffinityEnabled (dashboard default), which keeps
-    # a conversation's requests pinned to the same slot so there's a matching
-    # prefix to reuse.
-    # No published host port: llama-server is reached only via the omniroute
-    # gateway on the ai-stack docker network now — see issue #15. Its
-    # unauthenticated API no longer needs to be LAN-reachable directly.
-    expose:
-      - "8080"
-    restart: unless-stopped
-    networks: [ai-stack]
-    labels:
-      # ponytail: idle-timeout tuning lives here, not in a separate lazytainer config file —
-      # one place to look. Raise LAZYTAINER_INACTIVE_TIMEOUT if 15 min proves too eager.
-      - "lazytainer.group.llamaserver.sleepMethod=stop"
-      - "lazytainer.group.llamaserver.ports=8080"
-      - "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
-      - "lazytainer.group.llamaserver.minPacketThreshold=2"
-
-  # Dedicated backend for qwen-code's tool-call harmfulness classifier
-  # (fastModel in ~/.qwen/settings.json). Was aliased onto llama-server's own
-  # 27B connection — every classification call then queued behind whatever
-  # heavy generation was already running on that model's 2 GPU slots (issue
-  # tracker: OmniRoute semaphore/pr-agent investigation).
-  #
-  # Tried CPU-only first (own process avoids the GPU queue entirely) — too
-  # slow in practice: real classification calls blew past OmniRoute's 60s
-  # timeout and retry-looped (504→499→504...). Moved to GPU instead.
-  #
-  # Context sizing: qwen-code's classifier transcript is hard-capped in its
-  # own source (MAX_TRANSCRIPT_MESSAGES=40, MAX_HISTORICAL_ACTION_CHARS=4000
-  # per message, packages/core/src/permissions/classifier-transcript.ts) —
-  # worst case is ~40-50K tokens, nowhere near the 131072 originally set in
-  # settings.json (that number was copied from the main model's entry, not
-  # a real qwen-code requirement). 65536 ctx gives ~1.5x margin over that
-  # worst case. Qwen3-4B-Instruct-2507 is still the model choice — smallest
-  # Qwen3 with long native context (262144) without RoPE-scaling, in case
-  # that margin ever needs to grow.
-  #
-  # VRAM: weights+KV math (~4.9GiB) predicted comfortable headroom in the
-  # ~6.1GiB free on the R9700, but measured live it actually used ~5.85GiB —
-  # left only ~700MB free, too tight. Dropping --batch-size/--ubatch-size
-  # barely moved it (~768MB free) — wrong lever. Actual cause: llama-server
-  # runs with --flash-attn on but this service was missing it — without
-  # flash attention the unfused attention compute buffer at 65536 ctx is
-  # much larger (roughly O(n^2) intermediate buffers vs flash-attn's fused,
-  # near-linear workspace), dwarfing the naive weights+KV estimate. Added
-  # --flash-attn on to match llama-server; re-verify with rocm-smi after
-  # deploy before trusting any of these numbers again. GPU_MAX_HW_QUEUES=1
-  # carried over from llama-server's comment above — same ROCm/ROCm#5706
-  # clock-pinning bug applies now that two HIP contexts (this +
-  # llama-server) share the card.
-  #
-  # --reasoning off is a no-cost safety net, not a confirmed-needed fix:
-  # ggml-org/llama.cpp#20809 (closed) documents some server builds
-  # misdetecting Qwen3-Instruct-2507 models as thinking models, routing
-  # tool-call output into reasoning_content instead of tool_calls — exactly
-  # the failure mode that ruled out the 27B model for this role in the
-  # first place. Whether the current image build still has it was never
-  # independently confirmed (see docs/research/fast-model-choice.md §4/§6).
-  qwen-classifier:
-    image: ghcr.io/ggml-org/llama.cpp:server-rocm
-    container_name: qwen-classifier
-    devices:
-      - /dev/kfd
-      - /dev/dri
-    group_add:
-      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
-      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
-    security_opt:
-      - seccomp=unconfined
-    ipc: host
-    environment:
-      - GPU_MAX_HW_QUEUES=1
-    volumes:
-      - models:/models
-    command: >
-      -m /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}
-      --host 0.0.0.0
-      --port 8080
-      --n-gpu-layers 28
-      --ctx-size 65536
-      --parallel 1
-      --batch-size 512
-      --ubatch-size 128
-      --flash-attn on
-      --reasoning off
-      --cache-type-k q4_0
-      --cache-type-v q4_0
       --jinja
+    # see docs/architecture.md#llama-server (flags), docs/network-access.md#backends (no host port)
     expose:
       - "8080"
     restart: unless-stopped
     networks: [ai-stack]
 
-  # ponytail: one-off downloader, not a standing service — run via
+  # see docs/operations.md#model-downloads; the list of files is models.yml
-  # `docker compose --profile tools run --rm downloader`. Folded into
-  # scripts/update.sh, which runs this every time; the `test -f` guard is
-  # what makes that safe to re-run without re-downloading. Keeps the model
-  # files inside the named `models` volume instead of a host bind-mount.
   downloader:
-    image: curlimages/curl:latest
+    image: alpine:3
-    profiles: ["tools"]
-    # ponytail: named volume is created root-owned; curl_user (uid 100) can't
-    # write into it otherwise, so run as root for this one-off job.
-    user: root
-    volumes:
-      - models:/models
-    entrypoint: ["sh", "-c"]
-    command:
-      - >
-        test -f /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} &&
-        echo "already downloaded, skipping" ||
-        curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
-        https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
-
-  # Same pattern as downloader above, separate service so this one small
-  # file doesn't get re-checked/re-pulled by the big model's job.
-  downloader-classifier:
-    image: curlimages/curl:latest
     profiles: ["tools"]
     user: root
     volumes:
       - models:/models
-    entrypoint: ["sh", "-c"]
+      - ./models.yml:/models.yml:ro
+    entrypoint: ["sh", "-euo", "pipefail", "-c"]
     command:
-      - >
+      - |
-        test -f /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf} &&
+        apk add --no-cache -q curl yq-go
-        echo "already downloaded, skipping" ||
+        yq -r '.models[] | (.file // "") + " " + (.url // "")' /models.yml | while read -r file url; do
-        curl -L --fail --create-dirs -o /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}
+          if [ -z "$$url" ]; then echo "models.yml: entry '$$file' needs both file and url" >&2; exit 1; fi
-        https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}
+          if [ -f "/models/$$file" ]; then echo "$$file: already downloaded, skipping"; continue; fi
+          curl -L --fail -o "/models/$$file.part" "$$url"
+          mv "/models/$$file.part" "/models/$$file"
+        done
 
-  # Fetches the three Qwen-Image 2.1 (uncensored GGUF) files ComfyUI needs
+  # see docs/operations.md#speed-bench
-  # (diffusion model, text encoder, VAE) — same test -f guard pattern as
+  speed-bench:
-  # downloader above. See docs/research/image-generation-model-choice.md and
-  # issue #42.
-  #
-  # GGUF weights go in models/unet (ComfyUI-GGUF's own convention), not
-  # models/diffusion_models (that was the old FP8-safetensors layout) — see
-  # github.com/city96/ComfyUI-GGUF. Text encoder/VAE stay safetensors, loaded
-  # by ComfyUI's normal CLIPLoader/VAELoader, so their folders are unchanged.
-  #
-  # ponytail: target paths otherwise assume ComfyUI's standard models/ layout
-  # under BASE_STORAGE_PATH (/storage) — same "not independently confirmed
-  # against the image's Dockerfile" caveat flagged on the comfyui service
-  # below. If ComfyUI doesn't pick these up, check its actual models root
-  # first.
-  downloader-comfyui:
-    image: curlimages/curl:latest
     profiles: ["tools"]
-    user: root
+    build: ./speed-bench
+    entrypoint: ["python", "/app/speed_bench.py", "--url", "http://llama-server:8080"]
     volumes:
-      - comfyui-data:/storage
+      - hf-cache:/root/.cache/huggingface
-    entrypoint: ["sh", "-c"]
+      - ./bench-results:/results
-    command:
-      - >
-        mkdir -p /storage/models/unet /storage/models/text_encoders /storage/models/vae &&
-        (test -f /storage/models/unet/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen-image-2.1-UC-Q4_K_M.gguf} &&
-        echo "diffusion model already downloaded, skipping" ||
-        curl -L --fail --create-dirs -o /storage/models/unet/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen-image-2.1-UC-Q4_K_M.gguf}
-        https://huggingface.co/abenzerps/Qwen-Image-2.1-Uncensored-GGUF/resolve/main/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen-image-2.1-UC-Q4_K_M.gguf}) &&
-        (test -f /storage/models/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen3vl_8b_int8_convrot.safetensors} &&
-        echo "text encoder already downloaded, skipping" ||
-        curl -L --fail --create-dirs -o /storage/models/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen3vl_8b_int8_convrot.safetensors}
-        https://huggingface.co/abenzerps/Qwen-Image-2.1-Uncensored-GGUF/resolve/main/${COMFYUI_TEXT_ENCODER_FILE:-qwen3vl_8b_int8_convrot.safetensors}) &&
-        (test -f /storage/models/vae/${COMFYUI_VAE_FILE:-qwen_image_2.1_vae_bf16.safetensors} &&
-        echo "vae already downloaded, skipping" ||
-        curl -L --fail --create-dirs -o /storage/models/vae/${COMFYUI_VAE_FILE:-qwen_image_2.1_vae_bf16.safetensors}
-        https://huggingface.co/abenzerps/Qwen-Image-2.1-Uncensored-GGUF/resolve/main/${COMFYUI_VAE_FILE:-qwen_image_2.1_vae_bf16.safetensors})
-
-  # Local image generation — see issue #38 (wayfinder map). yurisasc's image
-  # is gfx1201-tuned specifically (R9700's arch), unlike the official/AMD
-  # ComfyUI image which doesn't pin RDNA4 support — see
-  # docs/research/image-generation-options.md.
-  #
-  # Model is Qwen-Image 2.1 GGUF (uncensored variant, see .env.example) —
-  # needs the ComfyUI-GGUF custom node (github.com/city96/ComfyUI-GGUF) for
-  # its "Unet Loader (GGUF)" node. Not confirmed pre-installed in this image;
-  # if missing, exec in and `git clone` it into custom_nodes, then pip
-  # install its requirements.txt and restart the container.
-  comfyui:
-    image: yurisasc/comfyui-rocm7.1:latest
-    container_name: comfyui
-    devices:
-      - /dev/kfd
-      - /dev/dri
-    # Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in
-    # .env.example and docs/research/rocm-gpu-pin-and-render-group.md.
-    group_add:
-      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
-      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
-    security_opt:
-      - seccomp=unconfined
-    ipc: host
-    environment:
-      - HSA_OVERRIDE_GFX_VERSION=12.0.1
-      - PYTORCH_ROCM_ARCH=gfx1201
-      # This image also wants GID env vars directly (its own README asks
-      # for both these and group_add above) — same HOST_VIDEO_GID/
-      # HOST_RENDER_GID resolved by scripts/update.sh, shared with
-      # llama-server now instead of comfyui-only vars.
-      - PUID=${COMFYUI_PUID}
-      - PGID=${COMFYUI_PGID}
-      - VIDEO_GID=${HOST_VIDEO_GID}
-      - RENDER_GID=${HOST_RENDER_GID}
-      - BASE_STORAGE_PATH=/storage
-    volumes:
-      - comfyui-data:/storage
-    # ponytail: exact internal storage path taken from the image's own
-    # BASE_STORAGE_PATH env var, not independently confirmed against its
-    # Dockerfile — if models/workflows don't persist across a recreate,
-    # check this against the image's actual entrypoint first.
-    #
-    # Published host port (unlike llama-server's ai-stack-only pattern):
-    # ComfyUI's own UI is meant to be reachable directly too, for a planned
-    # external nginx reverse-proxy route to comfy.home — not just through
-    # OmniRoute. Still also reachable at http://comfyui:8188 internally on
-    # ai-stack, which is the URL to register as OmniRoute's comfyui
-    # provider (dashboard or POST /api/providers, per docs/proxy-key-onboarding.md
-    # — same undocumented-in-repo manual flow already used for llama-server).
-    ports:
-      - "8138:8188"
-    restart: unless-stopped
     networks: [ai-stack]
 
-  # Replaces litellm — see issue #31 (wayfinder map) for the full migration
+  # see docs/architecture.md#omniroute; runtime config lives in its dashboard, see docs/gateway.md
-  # rationale/findings. No static config.yaml equivalent: provider routing
-  # (llama-server, searxng-search) is registered once through the dashboard
-  # or POST /api/providers after first boot, not checked into this repo —
-  # see docs/proxy-key-onboarding.md.
   omniroute:
     image: diegosouzapw/omniroute:latest
     container_name: omniroute
@@ -276,64 +77,28 @@ services:
       - omniroute-data:/app/data
     env_file: .env
     environment:
-      # Split-port mode: dashboard and API are fully separate ports (unlike
-      # LiteLLM's single :4000 for both /v1 and /ui) — both published
-      # directly below, unlike the old :4000-only host mapping.
       - API_HOST=0.0.0.0
       - API_PORT=${OMNIROUTE_API_PORT:-20129}
       - DASHBOARD_PORT=${OMNIROUTE_DASHBOARD_PORT:-20128}
-      # Required to register llama-server/searxng-search as providers —
+      # Security: relaxes OmniRoute's SSRF guard — see docs/gateway.md#provider-connections
-      # their base URLs are LAN/container-internal addresses, blocked by
-      # default (SSRF guard against public-provider spoofing).
       - OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS=true
       - OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS=true
-      # Required (production) per docs/reference/ENVIRONMENT.md — shared
-      # secret for the internal Codex Responses WebSocket bridge. Missed on
-      # first pass; docker-compose config validated fine without it, but
-      # the docs are explicit this one's required, not optional.
       - OMNIROUTE_WS_BRIDGE_SECRET=${OMNIROUTE_WS_BRIDGE_SECRET}
-      # Default heap (1024MB) is dashboard-only sized per OmniRoute's own
-      # Docker guide — every client here is a coding CLI, which needs the
-      # larger figure the guide recommends. Paired with mem_limit below.
       - OMNIROUTE_MEMORY_MB=8192
-      # Default 300000 (5 min) per OmniRoute's own docs, but this deployment
+      # see docs/gateway.md#timeouts (both timeouts below)
-      # had it dialed down elsewhere (dashboard) to ~95s — too tight for a
-      # contended local llama-server: large-context prefill under multiple
-      # concurrent slots can outrun that before the first SSE token arrives,
-      # so OmniRoute cancels a request that was actually still working (see
-      # LLAMA_PARALLEL above for the other half of this fix). Raised here so
-      # it's tracked in git instead of a dashboard-only setting.
       - STREAM_IDLE_TIMEOUT_MS=${OMNIROUTE_STREAM_IDLE_TIMEOUT_MS:-180000}
-      # Different timer than STREAM_IDLE_TIMEOUT_MS above — that one only
-      # bounds gaps *between* SSE chunks once streaming has started.
-      # REQUEST_TIMEOUT_MS bounds the wait for the *first* non-ping SSE
-      # event, and it's what was still firing ("Stream produced no non-ping
-      # SSE event within 95000ms") the morning after the timeout above was
-      # raised — see docs/research/omniroute-non-ping-sse-stream-timeout.md.
-      # Default 600000 (10 min) per OmniRoute's own docs, but the effective
-      # deadline is remaining budget after retries/cooldowns eat into it, not
-      # a flat timer, so raised well past the default for headroom.
       - REQUEST_TIMEOUT_MS=${OMNIROUTE_REQUEST_TIMEOUT_MS:-1800000}
-    # Same reasoning as litellm's extra_hosts entry below — ai-stack's bridge
+    # see docs/gateway.md#web-search
-    # network can't resolve search.home on its own.
     extra_hosts:
       - "search.home:${SEARXNG_LAN_IP}"
     ports:
       - "${OMNIROUTE_API_PORT:-20129}:${OMNIROUTE_API_PORT:-20129}"
       - "${OMNIROUTE_DASHBOARD_PORT:-20128}:${OMNIROUTE_DASHBOARD_PORT:-20128}"
-    # 10+ GiB ceiling per OmniRoute's Docker guide, matching
-    # OMNIROUTE_MEMORY_MB=8192 above.
     mem_limit: 10g
-    # SQLite WAL needs time to checkpoint back into the main DB file on
+    # Don't shorten: SQLite needs it to checkpoint its WAL, or recent writes are lost — see docs/architecture.md#omniroute
-    # shutdown — the Docker guide's --stop-timeout 40 equivalent.
     stop_grace_period: 40s
     restart: unless-stopped
     networks: [ai-stack]
-    # ponytail: TCP-connect check, not an HTTP /healthz GET — the image has
-    # no python3/curl/wget (confirmed live, `which` found only node), and
-    # OmniRoute's own Docker guide already treats a bare TCP probe on this
-    # port as an acceptable liveness check, not just the HTTP one. Simpler
-    # and avoids depending on /healthz's exact path/response shape.
     healthcheck:
       test:
         - CMD-SHELL
@@ -343,51 +108,14 @@ services:
       retries: 3
       start_period: 40s
 
-  lazytainer:
+  # see docs/architecture.md#qdrant, docs/network-access.md#qdrant
-    image: ghcr.io/vmorganp/lazytainer:master
-    container_name: lazytainer
-    # NOT network_mode: host — lazytainer identifies its own container by
-    # matching os.Hostname() against the Docker container-ID list
-    # (vmorganp/Lazytainer, configureFromLabels()); under host networking the
-    # container inherits the host's hostname instead of its own ID, so that
-    # match always fails and it panics with "Could not determine container ID
-    # of lazytainer" on every start. Host networking also can't see traffic
-    # to llama-server:8080 anyway — that port only exists on the ai-stack
-    # bridge network (no host port published, see issue #15 above). Joining
-    # ai-stack instead fixes both: hostname becomes the real container ID,
-    # and it's on the same network as the traffic it's watching.
-    networks: [ai-stack]
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    restart: unless-stopped
-    depends_on:
-      - llama-server
-
-  # RAG vector store — see docs/agents/... (wayfinder). Dashboard UI published
-  # directly like comfyui above, not gatewayed through omniroute (it isn't an
-  # LLM provider).
   qdrant:
     image: qdrant/qdrant:latest
     container_name: qdrant
     volumes:
       - qdrant-data:/qdrant/storage
     ports:
-      - "6333:6333"
+      - "127.0.0.1:6333:6333"
-    restart: unless-stopped
-    networks: [ai-stack]
-
-  # RAG graph store, native vector index too (can absorb qdrant's job later
-  # if the two-DB split proves unnecessary — see wayfinder notes).
-  neo4j:
-    image: neo4j:5-community
-    container_name: neo4j
-    environment:
-      - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD:?run scripts/update.sh first to resolve this}
-    volumes:
-      - neo4j-data:/data
-    ports:
-      - "7474:7474" # browser UI
-      - "7687:7687" # bolt
     restart: unless-stopped
     networks: [ai-stack]
 
@@ -397,6 +125,5 @@ networks:
 volumes:
   models:
   omniroute-data:
-  comfyui-data:
   qdrant-data:
-  neo4j-data:
+  hf-cache:

docs/agents/domain.md

@@ -28,6 +28,14 @@ When your output names a domain concept (in an issue title, a refactor proposal,
 
 If the concept you need isn't in the glossary yet, that's a signal — either you're inventing language the project doesn't use (reconsider) or there's a real gap (note it for `/domain-modeling`).
 
+## New or conflicting terms
+
+When you meet a term that isn't in `CONTEXT.md`:
+
+- **A synonym of an existing term** (e.g. "proxy" for *gateway*): use the canonical term. There's nothing to edit or ask.
+- **A genuinely new concept:** add it to `CONTEXT.md` in the same change, as the term, a one- or two-sentence definition and an _Avoid_ list, with no values or config. Name the new term in your final message so the user can veto it.
+- **A conflict with an existing definition** (the code or the user uses a term differently than `CONTEXT.md` defines it): stop and ask the user. Redefining a term is their decision.
+
 ## Flag ADR conflicts
 
 If your output contradicts an existing ADR, surface it explicitly rather than silently overriding:

docs/agents/writing-docs.md

@@ -0,0 +1,23 @@
+# Writing docs
+
+How to write and update this repo's docs and code comments. The source is the truth: docs explain *why* and point at the source for *what*.
+
+## Rules
+
+1. **Source-of-truth line.** Every doc outside `docs/research/` and `docs/agents/` opens with a `> **Source of truth:** …` line. It names the files, services or keys the doc explains and says the source wins on disagreement. In the body, name keys and flags (`LLAMA_PARALLEL`, `--cache-reuse`) and leave their values in the source. When a doc and its source disagree, fix the doc in the same change.
+2. **The README index is the doc map.** Every new doc gets a row in `README.md`'s Docs table.
+3. **Code comments are pointers.** In `docker-compose.yml`, `.env.example` and `scripts/`, a comment is a one-line `# see docs/<file>.md#<section>` pointer, and the explanation goes in that doc section. Two things stay inline as one line, next to the pointer:
+   - a **warning** about data loss, security, or an edit that breaks the stack or a script
+   - on `.env.example` keys that `update.sh` fills in, the marker "auto-filled by update.sh — leave blank"
+
+   A `ponytail:` note becomes a pointer too, with its reasoning moved into the doc.
+4. **A source change carries its doc.** When you change code that has a pointer, update the doc section it names in the same change.
+5. **`docs/research/` is frozen history.** New research goes in a new file plus a row in [`docs/research/README.md`](../research/README.md). When a research doc goes stale, change its status in that index and leave the file itself untouched.
+
+Terminology follows [`CONTEXT.md`](../../CONTEXT.md). For a term that isn't there, see [domain docs → New or conflicting terms](domain.md#new-or-conflicting-terms).
+
+## Done when
+
+- every pointer in the code you touched resolves to an existing heading
+- every doc you touched opens with its source-of-truth line
+- no doc you touched restates a value from its source

docs/architecture.md

@@ -0,0 +1,46 @@
+# Architecture
+
+> **Source of truth:** [`docker-compose.yml`](../docker-compose.yml) → each service below. This page explains *why*; flags, values and defaults live in the source. If they disagree, the source wins.
+
+Terms as defined in [`CONTEXT.md`](../CONTEXT.md).
+
+Every workload calls the **gateway** (`omniroute`) with its own API key. The gateway forwards each call over a **provider connection** to the one **backend** on the GPU: `llama-server` (main model). `qdrant` sits beside them on the same Docker network. It isn't a model, so it isn't behind the gateway.
+
+## GPU access
+
+`llama-server` carries these GPU settings in compose:
+
+- **`group_add` uses numeric GIDs, not group names.** Docker resolves a *named* `group_add` entry against the container's own `/etc/group`, not the host's. That fails unpredictably when the image doesn't define the group, and it gets worse when several GPU services start at once. `scripts/update.sh` resolves the host's real GIDs into `HOST_VIDEO_GID` / `HOST_RENDER_GID` (see [operations → Host-resolved values](operations.md#host-resolved-values)).
+- **`GPU_MAX_HW_QUEUES`** works around [ROCm/ROCm#5706](https://github.com/ROCm/ROCm/issues/5706): the card sits pinned at full load and boost clock whenever two HIP contexts touch it. Keep it set on every GPU service.
+
+Background: [`research/rocm-gpu-pin-and-render-group.md`](research/rocm-gpu-pin-and-render-group.md). Its service names are older than the current ones, but the findings still apply.
+
+## llama-server
+
+The main model's backend. It is always running. Lazytainer used to idle-stop it, but was removed because its idle detection could never fire behind the gateway ([`research/lazytainer-omniroute-idle-stop.md`](research/lazytainer-omniroute-idle-stop.md)).
+
+- **Model file** (`LLAMA_MODEL_FILE`): the quant choice and its VRAM budget are in [`research/qwen3.8-27b-quant.md`](research/qwen3.8-27b-quant.md).
+- **`LLAMA_GPU_LAYERS`**: the default puts every layer on the GPU. The model is dense, not Mixture-of-Experts, so this is the only RAM-offload knob. `--n-cpu-moe`, `--cpu-moe` and `--override-tensor "exps"` do nothing here. There is no SSD tier to enable either. llama.cpp mmaps the model file, so if GPU and RAM ever can't hold it, the OS pages the rest in from disk as a slow, implicit last resort. Explicit tiered offload is an [open llama.cpp request](https://github.com/ggml-org/llama.cpp/discussions/12507).
+- **`LLAMA_CTX_SIZE`**: set to the model's own maximum (`max_position_embeddings`). It only fits beside the weights because the KV cache is quantized (`LLAMA_KV_CACHE_TYPE`, used for both `--cache-type-k` and `--cache-type-v`). An fp16 KV cache at that size would not fit the card. The numbers are in the quant research doc.
+- **`LLAMA_PARALLEL`** is the backend's number of **slots**. It's kept low on purpose: more slots meant more contended prefill, which pushed requests past the gateway's timeouts ([gateway → Timeouts](gateway.md#timeouts)). Requests beyond the slot count wait in llama.cpp's own queue, which has no timeout. So the timeout that matters is the gateway's per-connection one.
+- **Per-slot context = `LLAMA_CTX_SIZE / LLAMA_PARALLEL`.** llama.cpp divides the context across slots, and each request gets one slot's share. Every coding CLI's context-window setting must use this number, not the raw context size. Real sessions already fill a large share of it, so check that the per-slot figure stays comfortably above observed usage before lowering either value.
+- **No `--cache-reuse`:** this model's context doesn't support it. llama-server logs "cache_reuse is not supported by this context, it will be disabled" at startup, so the flag was dropped. Re-prefill avoidance comes from `--cache-ram` (below) and OmniRoute's prompt-cache affinity (a dashboard setting, on by default), which keeps a conversation on the same slot. The earlier fix for the gateway's non-ping SSE timeout ([`research/omniroute-non-ping-sse-stream-timeout.md`](research/omniroute-non-ping-sse-stream-timeout.md)) assumed `--cache-reuse` worked, so long re-prefills can still hit that timeout.
+- **Testing for token corruption** (typos, broken tool-call keys): flip one key at a time in the server's `.env`, then run `docker compose up -d llama-server`. Blank `LLAMA_SPEC_ARGS` rules speculative decoding in or out, `LLAMA_KV_CACHE_TYPE=f16` does the same for the quantized KV cache (lower `LLAMA_CTX_SIZE` so it fits), and `LLAMA_CACHE_RAM=0` turns off the RAM prompt cache. A drafter's quant can't cause wrong tokens: the target model verifies every draft token, so a worse drafter only lowers acceptance.
+- **`LLAMA_CACHE_RAM`** (`--cache-ram`, MiB) is the host-RAM budget for saved prompt-cache entries. When a slot is reused, llama.cpp parks the old prompt's state in RAM so it can be restored instead of re-prefilled. The llama.cpp default is too small for ~40k-token agent prompts (entries are ~2 GiB each), so entries got evicted and requests re-prefilled in full. Raising it is the only place spare DDR RAM helps; it does not speed up decode. Keep it well below the host's free RAM. `--no-cache-idle-slots` stays on while [llama.cpp#27148](https://github.com/ggml-org/llama.cpp/issues/27148) is open: with the RAM cache on, an LRU-picked slot can keep an unrelated conversation's state, which showed up as misspelled output. With `--kv-unified`, turning off idle-slot caching closes that path and keeps the RAM cache ([diagnosis](research/inference-engine-spec-decoding-multi-gpu.md)). Evidence and the A/B method: [`research/qwen3.8-27b-throughput-tuning.md`](research/qwen3.8-27b-throughput-tuning.md).
+- **`--jinja`** enables the chat template that tool calling depends on. Without it, tool-use requests fail outright.
+- **Tool-calling reliability** of this model family on llama.cpp has open upstream parser bugs: [`research/qwen3.8-27b-tool-calling.md`](research/qwen3.8-27b-tool-calling.md).
+- **Speculative decoding (DFlash2):** all its flags live in one key, `LLAMA_SPEC_ARGS`: `--spec-type draft-dflash`, the drafter GGUF (`-md`, fetched by the downloader from [`models.yml`](../models.yml)) the draft length (`--spec-draft-n-max`), and `--spec-draft-p-min`, which ends a draft early once DFlash2's own confidence in the next token drops below the threshold. Long thinking output at the recommended sampling temperature gets low acceptance, so short confident drafts waste less verify work than long ones ([`common/speculative.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/common/speculative.cpp), DFlash2 lattice walk). One key means a single `.env` edit swaps the drafter or turns speculation off (leave it blank) for an A/B test. A blank server value differs from `.env.example`, so `update.sh` will ask about it on every run until it's restored ([operations → Env sync](operations.md#env-sync)). It replaced MTP (draft heads inside the target GGUF). Upstream: [llama.cpp#27342](https://github.com/ggml-org/llama.cpp/pull/27342). Draft acceptance can collapse at high slot counts ([llama.cpp#27117](https://github.com/ggml-org/llama.cpp/issues/27117)), another reason `LLAMA_PARALLEL` stays low.
+- **No classifier backend:** the separate `qwen-classifier` was removed to free VRAM for the drafter. Anything that needs a small "fast" model must use the main model.
+- **No host port**: see [network access → Backends](network-access.md#backends).
+
+## omniroute
+
+The gateway. Its runtime configuration (provider connections, API keys) lives in its own database and is managed through the dashboard, not in this repo: see [`gateway.md`](gateway.md).
+
+- **`OMNIROUTE_MEMORY_MB` + `mem_limit`**: the default heap is sized for dashboard-only use. OmniRoute's Docker guide recommends more for coding-agent traffic, and `mem_limit` is the matching container ceiling.
+- **`stop_grace_period`**: SQLite needs time on shutdown to checkpoint its WAL back into the main database file. If this is shortened, recent writes (keys, provider connections) can be lost.
+- **Healthcheck is a TCP connect**, not an HTTP probe. The image has only `node` (no curl, wget or python), and OmniRoute's Docker guide accepts a TCP probe as a liveness check.
+
+## qdrant
+
+A vector store. It has no authentication, so its port is bound to the server's loopback interface only: see [network access → qdrant](network-access.md#qdrant).

docs/coding-cli-setup/claude-code.md

@@ -2,12 +2,14 @@
 
 [← back to overview](index.md)
 
+> **Source of truth:** [Claude Code's docs](https://docs.claude.com/en/docs/claude-code/overview) for config keys; the [overview](index.md) for the gateway endpoint and context size. If this page disagrees with them, they win.
+
 Claude Code speaks the **Anthropic Messages API** — point it at the gateway's unified endpoint, not llama.cpp directly:
 
 ```bash
-export ANTHROPIC_BASE_URL=http://<ai-box>:${OMNIROUTE_PORT:-4000}
+export ANTHROPIC_BASE_URL=http://<server>:<OMNIROUTE_API_PORT>
-export ANTHROPIC_API_KEY=<claude-code-cli virtual key>
+export ANTHROPIC_API_KEY=<claude-code-cli API key>
 claude
 ```
 
-Requires llama.cpp's `--jinja` flag (already set in `docker-compose.yml`) — without it, tool-use requests fail outright.
+Relies on the backend's `--jinja` flag ([architecture → llama-server](../architecture.md#llama-server)).

docs/coding-cli-setup/index.md

@@ -1,34 +1,29 @@
-# Pointing a coding-agent CLI at this stack
+# Pointing a coding CLI at this stack
 
 [← back to README](../../README.md)
 
-This stack routes through the [AI gateway](https://git.arthurerlich.de/haylan/LLM-Server/issues/9) (OmniRoute, see issue #31) rather than talking to llama.cpp directly — llama.cpp's own port is internal-only now (see `docker-compose.yml`). The gateway exposes:
+> **Source of truth:** each CLI's own config docs (linked per page), plus [`docker-compose.yml`](../../docker-compose.yml) → `omniroute` for the gateway port. This page explains *how to wire them up*; ports and model settings live in those sources. If they disagree, the sources win.
 
-- **OpenAI-compatible**: `http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1`
+Terms as defined in [`CONTEXT.md`](../../CONTEXT.md). Every coding CLI is a workload: it reaches the models through the gateway, never a backend directly ([network access](../network-access.md)). The gateway speaks two wire formats for the same models:
-- **Anthropic Messages API** (OmniRoute's own `/v1/messages` endpoint, translating to the OpenAI-compatible backend): `http://<ai-box>:${OMNIROUTE_PORT:-4000}`
 
-Both serve the same underlying model — `Qwen3.8-27B-UD-Q4_K_XL.gguf`, registered in the gateway (naming is yours to pick when adding the llama-cpp provider connection — these docs assume `qwen3.8-27b-local` for continuity) — behind whichever wire format the client speaks.
+- **OpenAI-compatible:** `http://<server>:<OMNIROUTE_API_PORT>/v1`
+- **Anthropic Messages:** `http://<server>:<OMNIROUTE_API_PORT>` (OmniRoute's `/v1/messages`)
 
-`<ai-box>` is this machine's LAN address, or `proxy-ai.home` if your local DNS resolves that hostname directly to the box — see `docs/network-access.md`. If you're running a coding CLI from this machine itself, `localhost` works too.
+`<server>` is the server's LAN address or `proxy-ai.home`, or `localhost` on the server itself. The main model's name is whatever it was registered as in the gateway. These docs use `qwen3.8-27b-local` ([gateway → Provider connections](../gateway.md#provider-connections)).
 
-**Each CLI needs its own virtual key** — create one per docs/proxy-key-onboarding.md (omniroute's dashboard, `<workload>-<purpose>` naming, e.g. `claude-code-cli`, `kimi-cli`, `opencode-cli`). No budget set by default. These are the machine's interactive/high-priority workloads per `docs/proxy-request-priority.md`.
+**Each CLI needs its own API key**, labelled e.g. `claude-code-cli`, `kimi-cli`, `opencode-cli` ([gateway → API keys](../gateway.md#api-keys)).
 
-> **Read this before relying on it for real work.** Qwen3.8-27B's tool-calling has **documented, open llama.cpp upstream bugs** (parser fails on text before `<tool_call>`, tool calls emitted as inert XML inside thinking blocks — see `docs/research/qwen3.8-27b-tool-calling.md`). Every CLI below inherits this risk identically, regardless of wire format. Don't trust it for unattended multi-step agentic work until you've run the smoke test in [issue #5](https://git.arthurerlich.de/haylan/LLM-Server/issues/5) (and the proxy-specific smoke test in [issue #17](https://git.arthurerlich.de/haylan/LLM-Server/issues/17)).
+**Context window:** wherever a CLI asks for the main model's context size, use the per-slot context, `LLAMA_CTX_SIZE / LLAMA_PARALLEL` from the server's `.env`, not the raw context size ([architecture → llama-server](../architecture.md#llama-server)).
+
+> **Tool-calling risk.** The main model's tool calling has open llama.cpp upstream parser bugs ([`research/qwen3.8-27b-tool-calling.md`](../research/qwen3.8-27b-tool-calling.md)). Every CLI below inherits that risk, whatever its wire format. Test it on your own workflow before relying on it for unattended multi-step agent work.
 
 ## Per-CLI setup
 
-- [Claude Code CLI](claude-code.md)
+| CLI | Wire format | Config |
-- [Kimi CLI](kimi-cli.md)
+|---|---|---|
-- [OpenCode CLI](opencode.md)
+| [Claude Code](claude-code.md) | Anthropic Messages | `ANTHROPIC_BASE_URL` env var |
-- [Qwen Code CLI](qwen-code.md)
+| [Kimi CLI](kimi-cli.md) | OpenAI Chat Completions | `config.toml` provider block |
+| [OpenCode](opencode.md) | OpenAI Chat Completions | `opencode.json` provider block |
+| [Qwen Code](qwen-code.md) | OpenAI Chat Completions (main model) | `~/.qwen/settings.json` `modelProviders.openai` |
 
-## Summary
+If requests fail with a semaphore timeout under concurrent load, see [gateway → Timeouts](../gateway.md#timeouts).
-
-| CLI | Wire format | Endpoint | Config |
-|---|---|---|---|
-| [Claude Code](claude-code.md) | Anthropic Messages | `http://<ai-box>:${OMNIROUTE_PORT:-4000}` | `ANTHROPIC_BASE_URL` env var |
-| [Kimi CLI](kimi-cli.md) | OpenAI Chat Completions | `http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1` | `config.toml` provider block |
-| [OpenCode](opencode.md) | OpenAI Chat Completions | `http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1` | `opencode.json` provider block |
-| [Qwen Code](qwen-code.md) | OpenAI Chat Completions (2 models: chat + `fastModel`) | `http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1` | `~/.qwen/settings.json` `modelProviders.openai` |
-
-Further reading: `docs/research/qwen3.8-27b-tool-calling.md`, `docs/proxy-key-onboarding.md`, `docs/research/omniroute-account-semaphore-timeout.md` (a connection that can only handle a few concurrent requests — like `llama-server` or `qwen-classifier` — hits a hardcoded 30s reject once more requests queue up than its `maxConcurrent`, unless configured around it).

docs/coding-cli-setup/kimi-cli.md

@@ -2,13 +2,15 @@
 
 [← back to overview](index.md)
 
+> **Source of truth:** [Kimi CLI's repo](https://github.com/MoonshotAI/kimi-cli) for config keys; the [overview](index.md) for the gateway endpoint and context size. If this page disagrees with them, they win.
+
 Kimi CLI speaks plain **OpenAI Chat Completions**. Configure a provider block in its config file (`config.toml`):
 
 ```toml
 [providers.openai]
 type = "openai"
-base_url = "http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1"
+base_url = "http://<server>:<OMNIROUTE_API_PORT>/v1"
-api_key = "<kimi-cli virtual key>"
+api_key = "<kimi-cli API key>"
 ```
 
 If Kimi CLI's response parsing gets confused by Qwen's `<think>...</think>` reasoning tags, check its `reasoning_key` setting — it's configurable for non-standard local server responses.

docs/coding-cli-setup/opencode.md

@@ -2,30 +2,62 @@
 
 [← back to overview](index.md)
 
-Confirmed project: **`anomalyco/opencode`** (renamed from `sst/opencode` — don't confuse with the unrelated `opencode-ai/opencode` Go TUI). Docs: https://opencode.ai/docs/
+> **Source of truth:** [OpenCode v2 docs](https://opencode.ai/v2/docs) for config keys; the [overview](index.md) for the gateway endpoint and context size. If this page disagrees with them, they win. The v1 config is [deprecated](#v1-config-deprecated) but kept for installs that haven't migrated.
+
+OpenCode is **`anomalyco/opencode`** (renamed from `sst/opencode`). Don't confuse it with the unrelated `opencode-ai/opencode` Go TUI. It speaks plain **OpenAI Chat Completions**, so it talks to the gateway like any other OpenAI-compatible client.
+
+## Install
 
-**Install**:
 ```bash
-curl -fsSL https://opencode.ai/install | bash
+curl -fsSL https://opencode.ai/v2/install | bash
 ```
 
-**Config** (`opencode.json`, project root or `~/.config/opencode/opencode.json`):
+Homebrew, npm, bun, pnpm, yarn and AUR also work. See the [v2 docs](https://opencode.ai/v2/docs).
 
-```json
+## Config (v2)
+
+`opencode.json` or `opencode.jsonc`, in the project root (or `.opencode/`) or globally at `~/.config/opencode/opencode.json`. OpenCode searches upward from the current directory and merges, closest file wins.
+
+```jsonc
 {
   "$schema": "https://opencode.ai/config.json",
-  "provider": {
+  "providers": {
-    "aiproxy": {
+    "omniroute": {
-      "npm": "@ai-sdk/openai-compatible",
+      "name": "OmniRoute",
-      "name": "AI proxy (local)",
+      "package": "aisdk:@ai-sdk/openai-compatible",
-      "options": {
+      "settings": {
-        "baseURL": "http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1",
+        "baseURL": "https://<YOUR-OMNIROUTE-PROXY>/v1",
-        "apiKey": "<opencode-cli virtual key>"
+        "apiKey": "<token>"
       },
       "models": {
-        "qwen3.8-27b-local": {
+        "qwen3.8-27b": {
+          "modelID": "qwen3.8-27b-local//models/Qwen3.8-27B-UD-Q4_K_XL.gguf",
           "name": "Qwen3.8-27B",
-          "limit": { "context": 131072, "output": 8192 }
+          "capabilities": { "tools": true, "input": ["text"], "output": ["text"] },
+          "compatibility": { "reasoningField": "reasoning_content" },
+          "limit": { "context": 260000, "output": 32000 },
+          "cost": { "input": 0.4, "output": 3 },
+          "body": {
+            "temperature": 1,
+            "top_p": 0.95,
+            "top_k": 20,
+            "min_p": 0,
+            "presence_penalty": 0,
+            "chat_template_kwargs": { "reasoning_effort": "xhigh" }
+          },
+          "variants": [
+            { "id": "medium", "body": { "chat_template_kwargs": { "reasoning_effort": "medium" } } },
+            { "id": "fast", "body": { "chat_template_kwargs": { "reasoning_effort": "low" } } },
+            {
+              "id": "instruct",
+              "body": {
+                "temperature": 0.7,
+                "top_p": 0.8,
+                "presence_penalty": 1.5,
+                "chat_template_kwargs": { "enable_thinking": false }
+              }
+            }
+          ]
         }
       }
     }
@@ -33,13 +65,75 @@ curl -fsSL https://opencode.ai/install | bash
 }
 ```
 
-Set `limit.context` to the *per-slot* context this stack actually serves — `LLAMA_CTX_SIZE / LLAMA_PARALLEL` from `.env` (262144 / 2 = 131072 by default), not raw `LLAMA_CTX_SIZE` and not a value assumed from the model card: llama.cpp divides `--ctx-size` across concurrent slots, so each request only gets one slot's share. OpenCode uses this for its own context-management bookkeeping, not the server.
+### What each part does
 
-Select the model with `aiproxy/qwen3.8-27b-local`.
+| Key | Purpose |
+|---|---|
+| `providers` | Plural in v2 (v1: `provider`). The key (`omniroute`) is the prefix in model references. |
+| `package` | The adapter for the API shape. This stack uses an OpenAI-compatible one. |
+| `settings.baseURL` | The gateway URL, up to and including `/v1`. |
+| `settings.apiKey` | The gateway API key issued to OpenCode ([gateway → API keys](../gateway.md#api-keys)). The v2 docs' own examples use an `env` list instead, to keep the key out of the file. |
+| `models.<key>` | The short name you select. `modelID` is the id the gateway exposes, which the key doesn't have to match. |
+| `capabilities` | Declares tool calling and input/output modalities. |
+| `compatibility.reasoningField` | Where the backend returns reasoning text. llama.cpp uses `reasoning_content`. |
+| `limit` | OpenCode's own context bookkeeping. The server doesn't read it. See [overview](index.md) for the number to use. |
+| `body` | Extra request fields, sent on every call. Sampling values and `chat_template_kwargs` go here. |
+| `variants` | Named overrides on top of `body`. |
 
-**OpenCode-specific risks** (on top of the shared Qwen3.8-27B tool-calling risk — see [overview](index.md)):
+### Selecting a model and variant
-- Requires llama.cpp's `--jinja` flag (already set) — without it, OpenCode's unconditional tool-call scaffolding gets a 500.
-- [anomalyco/opencode#20669](https://github.com/anomalyco/opencode/issues/20669) (closed as "not planned" — a live, unfixed risk): OpenCode's `bash` tool crashes if the model omits the optional `description` field on a tool call; some local backends return `finish_reason: tool_calls` with an empty array, which can hang the agent loop instead of stopping cleanly.
-- Thinking-mode handling (`options.reasoningEffort`) is undocumented for models that emit inline `<think>` tags rather than a native reasoning API field — expect no effect from that config on this model; untested.
 
-Further reading: `docs/research/opencode-cli-setup.md`.
+Format: `provider/model#variant`. The provider ends at the first `/`.
+
+```bash
+opencode run --model omniroute/qwen3.8-27b          # default: xhigh reasoning
+opencode run --model omniroute/qwen3.8-27b#medium
+opencode run --model omniroute/qwen3.8-27b#fast     # low reasoning effort
+opencode run --model omniroute/qwen3.8-27b#instruct # thinking off, instruct sampling
+```
+
+### Notes
+
+- **Unverified keys:** the v2 docs show `"package": "@opencode/ai/providers/openai-compatible"` for custom providers, while this config uses the `aisdk:@ai-sdk/openai-compatible` form. The docs pages I could read don't confirm that prefix. If OpenCode rejects the provider, switch `package` to the documented value.
+- **Variants:** the v2 docs' own variant example puts settings under `settings`. This config uses `body`, which the models page lists as valid for provider-specific request fields.
+- **`limit.context`:** [overview](index.md) explains how per-slot context is derived. The example above uses a larger value than a plain per-slot division gives.
+
+## OpenCode-specific risks
+
+On top of the shared Qwen3.8-27B tool-calling risk (see [overview](index.md)):
+
+- Relies on the backend's `--jinja` flag ([architecture → llama-server](../architecture.md#llama-server)). Without it, OpenCode's unconditional tool-call scaffolding gets a 500.
+- [anomalyco/opencode#20669](https://github.com/anomalyco/opencode/issues/20669) (closed as "not planned", so a live, unfixed risk): OpenCode's `bash` tool crashes if the model omits the optional `description` field on a tool call. Some local backends return `finish_reason: tool_calls` with an empty array, which can hang the agent loop instead of stopping it.
+- Reasoning control goes through `chat_template_kwargs` in `body`, not OpenCode's native `reasoningEffort`. The native key has no effect against llama.cpp.
+
+## V1 config (deprecated)
+
+Use this only on OpenCode v1. It is no longer maintained here. Differences from v2: `provider` (singular), `npm` instead of `package`, `options` instead of `settings`, and a flat `reasoning` / `interleaved` block instead of `capabilities` / `compatibility`, and no `variants` or `body` in this config.
+
+```jsonc
+{
+  "$schema": "https://opencode.ai/config.json",
+  "model": "omniroute/qwen3.8-27b-local//models/Qwen3.8-27B-UD-Q4_K_XL.gguf",
+  "provider": {
+    "omniroute": {
+      "npm": "@ai-sdk/openai-compatible",
+      "name": "OmniRoute",
+      "options": { "baseURL": "https://<YOUR-OMNIROUTE-PROXY>/v1" },
+      "models": {
+        "qwen3.8-27b-local//models/Qwen3.8-27B-UD-Q4_K_XL.gguf": {
+          "name": "Qwen3.8-27B",
+          "reasoning": true,
+          "interleaved": { "field": "reasoning_content" },
+          "limit": { "context": 131000, "output": 65536 },
+          "cost": { "input": 0.4, "output": 3.0, "cache": { "read": 0.04, "write": 0 } }
+        }
+      }
+    }
+  }
+}
+```
+
+Select the model with `omniroute/<model-key>`. The v1 install script is `curl -fsSL https://opencode.ai/install | bash`.
+
+## Further reading
+
+[`research/opencode-cli-setup.md`](../research/opencode-cli-setup.md) (v1-era, base URL predates the gateway), [`research/opencode-auto-compact.md`](../research/opencode-auto-compact.md).

docs/coding-cli-setup/qwen-code.md

@@ -2,16 +2,13 @@
 
 [← back to overview](index.md)
 
-Qwen Code speaks plain **OpenAI Chat Completions**, and — unlike the other CLIs — needs *two* models: the main chat model, and a `fastModel` for Auto Mode's action classifier. Both are registered as separate providers in OmniRoute but reachable through the same gateway URL.
+> **Source of truth:** [Qwen Code's repo](https://github.com/QwenLM/qwen-code) for config keys; the [overview](index.md) for the gateway endpoint and context size. If this page disagrees with them, they win.
 
-## Why a second model exists
+Qwen Code speaks plain **OpenAI Chat Completions**. It uses the main model only. There is no separate classifier backend any more ([architecture → llama-server](../architecture.md#llama-server)).
 
-Auto Mode's action classifier (`permissions.autoMode`) is qwen-code's per-tool-call safety gate — it decides whether to auto-approve or block a shell command / tool call before it runs. It was originally aliased onto the main 27B model's own OmniRoute connection. That broke two ways in practice (see `docs/research/fast-model-choice.md` for the model research, and the issue-tracker history for the full incident):
+## Auto Mode's classifier (`fastModel`)
 
-- **Queued behind heavy work.** Every classification call competed for the main model's 2 GPU slots with whatever real generation was already running, so a classifier check could sit blocked for minutes.
+Auto Mode's action classifier (`permissions.autoMode`) is qwen-code's safety gate for every tool call. It calls `fastModel`. With no classifier backend, either point `fastModel` at the main model (it then queues behind the main backend's slots, the reason the separate backend once existed), or leave `fastModel` unset and don't use Auto Mode.
-- **CPU-only was tried first and was too slow.** Isolating the classifier onto its own CPU-only llama.cpp instance avoided the GPU queue entirely, but real classification calls (which can carry a non-trivial conversation transcript, not just the bare tool call) blew past OmniRoute's request timeout and retry-looped.
-
-The fix: a dedicated, GPU-resident `qwen-classifier` service (`docker-compose.yml`) running a small model (`Qwen3-4B-Instruct-2507`) on its own **partial** GPU offload — enough layers on the R9700 to be fast, sized to leave real VRAM headroom next to the 27B model rather than trusting a naive weights+KV estimate (see that service's comment block in `docker-compose.yml` for the actual measured numbers and the two wrong turns — batch-size tuning, then flash-attn — before partial offload turned out to be the real lever).
 
 ## `~/.qwen/settings.json`
 
@@ -23,39 +20,27 @@ The fix: a dedicated, GPU-resident `qwen-classifier` service (`docker-compose.ym
         "id": "<main-model-provider-id-in-omniroute>",
         "name": "qwen3.8-27b-local",
         "envKey": "OMNIROUTE_API_KEY",
-        "baseUrl": "http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1",
+        "baseUrl": "http://<server>:<OMNIROUTE_API_PORT>/v1",
-        "generationConfig": { "contextWindowSize": 131072 }
+        "generationConfig": { "contextWindowSize": <LLAMA_CTX_SIZE / LLAMA_PARALLEL> }
-      },
-      {
-        "id": "<classifier-provider-id-in-omniroute>",
-        "name": "qwen3-4b-classifier",
-        "envKey": "OMNIROUTE_API_KEY",
-        "baseUrl": "http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1",
-        "generationConfig": {
-          "contextWindowSize": 65536,
-          "extra_body": { "chat_template_kwargs": { "enable_thinking": false } }
-        }
       }
     ]
   },
   "security": { "auth": { "selectedType": "openai" } },
   "model": {
     "name": "<main-model-provider-id-in-omniroute>",
-    "baseUrl": "http://<ai-box>:${OMNIROUTE_PORT:-4000}/v1"
+    "baseUrl": "http://<server>:<OMNIROUTE_API_PORT>/v1"
   },
-  "fastModel": "<classifier-provider-id-in-omniroute>"
+  "fastModel": "<main-model-provider-id-in-omniroute>"
 }
 ```
 
-- `envKey` names the environment variable Qwen Code reads the virtual key from — set `OMNIROUTE_API_KEY=<qwen-code-cli virtual key>` before launching. Both providers can share one virtual key (as above); split it into two if you want separate usage tracking for chat vs. classifier calls.
+- `envKey` names the environment variable Qwen Code reads the API key from — set `OMNIROUTE_API_KEY=<qwen-code-cli API key>` before launching.
-- **`contextWindowSize` for the main model is per-slot, not `LLAMA_CTX_SIZE` itself** — llama.cpp divides `--ctx-size` across `LLAMA_PARALLEL` concurrent slots, and each request only gets one slot's share (same correction applies to OpenCode's `limit.context`). Compute it from `.env`: `LLAMA_CTX_SIZE / LLAMA_PARALLEL` = `262144 / 2` = **131072**.
+- **The main model's `contextWindowSize` is the per-slot context**, not `LLAMA_CTX_SIZE` itself ([overview](index.md)).
-- **The classifier's `contextWindowSize` (65536) is not per-slot math** — `qwen-classifier` runs `--parallel 1`, so its whole `--ctx-size` belongs to the one slot. 65536 isn't a guess either: qwen-code's own source hard-caps the classifier transcript (`MAX_TRANSCRIPT_MESSAGES=40`, `MAX_HISTORICAL_ACTION_CHARS=4000`/message in `packages/core/src/permissions/classifier-transcript.ts`) — worst case is ~40-50K tokens, so 65536 gives real margin without wasting VRAM the way the original 131072 (copied from the main model's entry, not an actual qwen-code requirement) would have.
-- `enable_thinking: false` on the classifier matters for parseability, though `Qwen3-4B-Instruct-2507` is already architecturally non-thinking (see `fast-model-choice.md` §3) — this is belt-and-suspenders for any future fast-model swap that isn't.
 - Qwen Code also recognizes `advisorModel`, `visionModel`, `compactionModel`, `imageModel` for other model roles — none are wired up in this stack; only `fastModel` is required.
 
 ## Web search via OmniRoute
 
-Qwen Code's own built-in web search (`tools.webSearch.enabled`) has nothing to search with here — leave it `false`. Instead this stack's SearXNG-backed search (README §"Web search") is exposed through a thin stdio MCP wrapper around OmniRoute's `/v1/search` REST endpoint (that endpoint isn't itself MCP — OmniRoute's real MCP surface is admin-only/LOCAL_ONLY-gated). Save this as e.g. `~/.qwen/mcp-servers/omniroute-search/index.mjs` (needs `@modelcontextprotocol/sdk` and `zod`: `npm init -y && npm i @modelcontextprotocol/sdk zod` in that directory):
+Qwen Code's own built-in web search (`tools.webSearch.enabled`) has nothing to search with here — leave it `false`. Instead this stack's SearXNG-backed search ([gateway → Web search](../gateway.md#web-search)) is exposed through a thin stdio MCP wrapper around OmniRoute's `/v1/search` REST endpoint (that endpoint isn't itself MCP — OmniRoute's real MCP surface is admin-only/LOCAL_ONLY-gated). Save this as e.g. `~/.qwen/mcp-servers/omniroute-search/index.mjs` (needs `@modelcontextprotocol/sdk` and `zod`: `npm init -y && npm i @modelcontextprotocol/sdk zod` in that directory):
 
 ```js
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
@@ -104,13 +89,13 @@ Register it in `~/.qwen/settings.json`:
 }
 ```
 
-It reuses the same `OMNIROUTE_API_KEY` env var as the model providers above — the virtual key needs search permission in OmniRoute, not just chat-completions.
+It reuses the same `OMNIROUTE_API_KEY` env var as the model providers above — the API key needs search permission in OmniRoute, not just chat-completions.
 
 **Non-interactive mode (`qwen -p ...`) needs this tool explicitly allow-listed.** MCP tools require interactive confirmation by default; `--approval-mode auto` alone doesn't bypass that for a non-interactive run — pass `--allowed-tools mcp__omniroute-search__search` (or `-y` for full YOLO) alongside `-p`, or the search call never reaches the classifier at all and silently no-ops. Confirmed live: without the allow-list, only the tool calls the CLI's non-interactive gate lets through end up as classifier requests.
 
 ## Auto Mode tuning
 
-Auto Mode's action classifier calls the fast model above. Even on the dedicated GPU-resident instance, give it real timeout headroom rather than trusting OmniRoute's default — and since this stack is a single trusted local proxy, it's reasonable to pre-approve requests to it rather than confirm every call:
+Auto Mode's action classifier calls `fastModel`, now the main model, so it can queue behind a heavy generation. Give it real timeout headroom rather than trusting OmniRoute's default. And since the gateway is a single trusted local endpoint, it's reasonable to pre-approve requests to it rather than confirm every call:
 
 ```json
 {
@@ -125,6 +110,6 @@ Auto Mode's action classifier calls the fast model above. Even on the dedicated
 
 `hints.allow` entries are free-text descriptions the classifier matches against, not exact strings — capped at 150 entries/200 chars each.
 
-Also set a generous per-connection timeout on the classifier's own OmniRoute provider connection (`providerSpecificData.timeoutMs`, dashboard or `PATCH /api/providers/{id}` — not a `.env` value, see `docs/network-access.md` for reaching the dashboard API). 120000ms is comfortable for the current GPU-resident setup (real measured latency: well under a second for a short check, low seconds for the largest realistic transcript) — this isn't the 20-minute figure the main 27B connection needs, since the classifier isn't competing for a contended GPU slot the way the main model can.
+The main model's provider connection already carries the long per-connection timeout (`providerSpecificData.timeoutMs`, see [gateway → Timeouts](../gateway.md#timeouts)), which now covers classifier calls too.
 
 Everything else in `~/.qwen/settings.json` (`hooks`, `security.auth`'s underlying tooling, editor prefs) is per-machine, not part of pointing at this stack — don't copy it wholesale between machines.

docs/gateway.md

@@ -0,0 +1,62 @@
+# Gateway (OmniRoute)
+
+> **Source of truth:** [`docker-compose.yml`](../docker-compose.yml) → `omniroute` service, and [`.env.example`](../.env.example) → OmniRoute section. Provider connections and API keys live in OmniRoute's own database, managed through its dashboard, not in this repo. This page explains *why* and *how*; values live in those sources. If they disagree, the source wins.
+
+Terms as defined in [`CONTEXT.md`](../CONTEXT.md). How the gateway fits the rest of the stack: [`architecture.md`](architecture.md#omniroute).
+
+## Dashboard
+
+Browse to `http://<server>:<OMNIROUTE_DASHBOARD_PORT>` from the LAN. The dashboard isn't routed to the internet: see [network access → Gateway](network-access.md#gateway). For the first login, use `OMNIROUTE_INITIAL_PASSWORD` from the server's `.env`, then change the password in the dashboard.
+
+## Provider connections
+
+Each backend is registered once in the dashboard (Providers → add a **llama.cpp** provider) or via `POST /api/providers`. The base URL is `http://<service>:<container port>` on the stack network, using the service name and port from compose. None of this is checked into the repo. These docs assume the main model is registered as `qwen3.8-27b-local`.
+
+- **`OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS` / `OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS`** have to be on, because backend URLs are private addresses. They relax OmniRoute's SSRF guard, which normally blocks registering a provider at a private or local address. Anyone with dashboard access can therefore point the gateway at any host on the stack network or the LAN.
+- **Stale classifier connection:** the `qwen-classifier` backend no longer exists. Delete its provider connection in the dashboard, or repoint it at the main model. If repointed, turn OmniRoute's memory-tool injection off for it ([`research/omniroute-memory-tool-injection-breaks-classifier.md`](research/omniroute-memory-tool-injection-breaks-classifier.md)).
+- Background on registering local backends: [`research/omniroute-qwen-websearch.md`](research/omniroute-qwen-websearch.md).
+
+## API keys
+
+Every workload gets its own API key, so each has its own visible usage.
+
+**Create**
+
+1. [Open the dashboard](#dashboard) → "Keys" → "Create API key".
+2. Label it `<workload>-<purpose>`, e.g. `paperless-ocr`, `claude-code-cli`. The dashboard lists keys by label, so the label list is the ledger. There's no separate tracking doc.
+3. Copy the key value. OmniRoute shows it only once.
+
+Keys can only be minted by hand for now. `POST /api/keys` needs a dashboard login session (`ManagementSessionAuth`), not a static bearer key, and that flow hasn't been verified against a live instance.
+
+**Hand over:** put the key in that workload's own config or `.env`, never in this repo. A key used for [web search](#web-search) needs search permission, not just chat completions.
+
+**Revoke** ("Keys" → delete) when the workload is retired or the key may have leaked, then remove it from the workload's config. There's no scheduled rotation.
+
+## Timeouts
+
+A slow request can be cut off at several layers. Each setting is named below, and its value lives where the list says:
+
+- **`STREAM_IDLE_TIMEOUT_MS`** (`.env.example`): the longest allowed gap *between* SSE chunks once streaming has started.
+- **`REQUEST_TIMEOUT_MS`** (`.env.example`): the wait budget for the *first* non-ping SSE event. Retries and cooldowns eat into it, so the effective deadline is whatever budget remains, not a flat timer. See [`research/omniroute-non-ping-sse-stream-timeout.md`](research/omniroute-non-ping-sse-stream-timeout.md).
+- **Per-connection `providerSpecificData.timeoutMs`** (dashboard only, or `PATCH /api/providers/{id}`): bounds a request that is waiting behind the backend's busy slots.
+- **Per-connection concurrency semaphore**: a hardcoded reject once more requests queue than the connection's `maxConcurrent`. It is not configurable, only avoidable: [`research/omniroute-account-semaphore-timeout.md`](research/omniroute-account-semaphore-timeout.md).
+- **`OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS`** (not set in this repo): [`research/omniroute-direct-response-timeout-outage-2026-09-15.md`](research/omniroute-direct-response-timeout-outage-2026-09-15.md).
+
+These are raised above OmniRoute's defaults because contended prefill on the main backend can take a long time before the first token ([architecture → llama-server](architecture.md#llama-server)).
+
+## Web search
+
+SearXNG runs elsewhere on the LAN as `search.home`. It's registered as a search provider connection and exposed through the gateway's `/v1/search` endpoint. The container can't resolve `search.home` (it's a LAN DNS name), so compose adds an `extra_hosts` entry from `SEARXNG_LAN_IP`, which `update.sh` resolves on the server ([operations → Host-resolved values](operations.md#host-resolved-values)).
+
+- Wiring it into Qwen Code: [`coding-cli-setup/qwen-code.md`](coding-cli-setup/qwen-code.md#web-search-via-omniroute).
+- Research: [`research/omniroute-qwen-websearch.md`](research/omniroute-qwen-websearch.md) (current setup), [`research/dashscope-websearch-selfhost-options.md`](research/dashscope-websearch-selfhost-options.md) (why DashScope isn't emulated).
+
+## Upstream caution
+
+OmniRoute's own docs (`docs/security/STEALTH_GUIDE.md`, `MITM-TPROXY-DECRYPT.md` and `PUBLIC_CREDS.md` in [its repo](https://github.com/diegosouzapw/OmniRoute)) describe shipped features for:
+
+- evading AI-provider client detection
+- intercepting HTTPS system-wide via a locally installed root CA
+- hiding credentials from secret scanners
+
+This stack's configuration uses none of them. They are a real characteristic of the upstream project, though, so weigh them before extending this integration.

docs/network-access.md

@@ -1,24 +1,29 @@
-# Network access: proxy-ai.home / proxy-ai.haylan.ch
+# Network access
 
-This stack has no chat UI — every client is a coding CLI reaching the AI gateway (OmniRoute). It doesn't run its own reverse proxy — it publishes the gateway's API port to the host and relies on the **existing Nginx Proxy Manager (NPM)** instance already fronting other self-hosted services on this network.
+> **Source of truth:** [`docker-compose.yml`](../docker-compose.yml) → the `ports:` / `expose:` entries of each service. This page explains *who can reach what, and why*; port numbers live in the source. If they disagree, the source wins.
 
-## llama.cpp's raw API stays LAN-only — deliberately
+Terms as defined in [`CONTEXT.md`](../CONTEXT.md). The stack doesn't run its own reverse proxy. The existing **reverse proxy** (Nginx Proxy Manager, not part of this stack) publishes the gateway under hostnames.
 
-The inference API (port `${LLAMA_PORT:-8080}`) is **not** registered in NPM and is **not** reachable externally. It has no authentication of its own — putting it on the public internet would mean an unauthenticated inference endpoint. Coding-agent CLIs (Claude Code, Kimi, OpenCode, Qwen Code — see `docs/coding-cli-setup/`) don't reach it directly at all now; they go through the gateway below, same as everything else.
+| What | Reachable from |
+|---|---|
+| Gateway API | LAN, and the internet via the reverse proxy |
+| Gateway dashboard | LAN only |
+| Backends | Stack network only (through the gateway) |
+| qdrant | The server itself only |
 
-If you later want external CLI access too, that's a deliberate scope change — see the map ([issue #1](https://git.arthurerlich.de/haylan/LLM-Server/issues/1)) before doing it, since it changes the security posture.
+## Gateway
 
-## The AI gateway (OmniRoute) — `proxy-ai.home` / `proxy-ai.haylan.ch`
+- **API port (`OMNIROUTE_API_PORT`)**: published on the server's LAN address. Two reverse-proxy hosts point at it:
+  - `proxy-ai.home`: LAN only.
+  - `proxy-ai.haylan.ch`: internet, through the DMZ that already forwards to the reverse proxy, which also manages the TLS certificate.
 
-As of [issue #31](https://git.arthurerlich.de/haylan/LLM-Server/issues/31) (migrated from LiteLLM), the gateway is OmniRoute:
+  Every call needs an [API key](gateway.md#api-keys), so the reverse proxy adds no auth of its own.
+- **Dashboard port (`OMNIROUTE_DASHBOARD_PORT`)**: published on the LAN only. It's never registered in the reverse proxy and has no port-forward, so it isn't reachable from the internet. Keep it that way: the dashboard manages every API key.
 
-- **`proxy-ai.home`** and **`proxy-ai.haylan.ch`** both point only at `${OMNIROUTE_PORT:-4000}` — the API port. Set up as two NPM Proxy Hosts pointing at this machine's LAN IP on that port; `proxy-ai.home` internal-only, `proxy-ai.haylan.ch` external via the DMZ already forwarding to NPM (let NPM issue/manage the TLS cert as usual).
+## Backends
-- The **dashboard** (`${OMNIROUTE_DASHBOARD_PORT:-20128}`) is never registered in NPM at all, and `docker-compose.yml` never publishes that port to the host either — it manages every workload's keys, so it doesn't belong on the public internet, same reasoning as LiteLLM's old `/ui`. Unlike LiteLLM, OmniRoute's split-port mode means this is structural (no network route exists) rather than an NPM path-deny rule that has to be maintained and could be misconfigured. Reach the dashboard only from the host itself or over SSH port-forward.
 
-**Every gateway call already requires a valid API key** (Bearer token, see `docs/proxy-key-onboarding.md`), so no extra NPM-level auth is needed for the external hostname.
+`llama-server` publishes no host port (`expose` only). It has no authentication, so it's reachable only on the stack's Docker network, through the gateway. Making it directly reachable would put an unauthenticated inference endpoint on the network. That's a deliberate security change, not a config tweak.
 
-## RAG knowledge graph (Neo4j) — `knowledge.proxy-ai.home`
+## qdrant
 
-Set up as an NPM Proxy Host pointing at this machine's LAN IP on Neo4j's Browser port (`7474`, see `docker-compose.yml`'s `neo4j` service, [PR #50](https://git.arthurerlich.de/haylan/LLM-Server/pulls/50)). Internal-only, same as `proxy-ai.home` — no DMZ/external route, this is admin/dev tooling, not a client-facing endpoint. Bolt (`7687`, the actual query protocol) isn't proxied through NPM at all — clients on the LAN reach it directly at `<this-machine>:7687`.
+Bound to the server's loopback interface, because qdrant has no authentication. From another machine, use an SSH port-forward to the server.
-
-Qdrant's dashboard (`6333`) stays on its raw LAN IP/port for now — no hostname assigned yet.

docs/operations.md

@@ -0,0 +1,73 @@
+# Operations
+
+> **Source of truth:** [`scripts/update.sh`](../scripts/update.sh), [`scripts/redownload-model.sh`](../scripts/redownload-model.sh), [`.env.example`](../.env.example), [`models.yml`](../models.yml), and the `speed-bench` service in [`docker-compose.yml`](../docker-compose.yml) with its [`speed-bench/Dockerfile`](../speed-bench/Dockerfile) and [`scripts/bench.sh`](../scripts/bench.sh). This page explains *what the script does and why*; the steps and defaults live in the source. If they disagree, the source wins.
+
+The stack runs on the **server**. After any change to the repo, run `./scripts/update.sh` there: it is the one command that brings the running stack in sync. It's safe to re-run at any time, and it only recreates what changed.
+
+## Self-update
+
+`git pull` runs before anything else. A script that updates itself isn't atomic against its own file changing mid-run. Pulling later once executed a mix of old and new script and compose content in a single pass. Keep the pull first.
+
+## Env sync
+
+`.env` is created from `.env.example` if it's missing. After that:
+
+- **Keys with a real default in `.env.example`** are tracked config, synced every run. A key missing from `.env` is added. A key whose value matches is left alone. A key whose value **differs** is a conflict:
+  - Interactively, all conflicts appear on one screen ([gum](#gum)). Selected keys take the new value, and unselected keys keep the server's value.
+  - Without a TTY (cron, CI, a pipe), any conflict is an error unless `--force` is passed, which accepts every new value.
+
+  To change a default for good, change `.env.example`, not the server's `.env`.
+- **Keys left blank in `.env.example`** ([secrets](#secrets) and [host-resolved values](#host-resolved-values)) are never synced. They're only filled when blank.
+- Keys removed from `.env.example` stay in the server's `.env`: the script adds keys and never prunes them. They're harmless, and you can delete them by hand.
+
+There's no rollback. If a bad config lands, `git revert` and re-run the script.
+
+## Secrets
+
+Random secrets are generated with `openssl` when blank and never overwritten after that. **`OMNIROUTE_STORAGE_ENCRYPTION_KEY` must never change after the first run**: it encrypts OmniRoute's whole database, and changing it makes the existing data unreadable. `OMNIROUTE_WS_BRIDGE_SECRET` is required by OmniRoute in production (its `docs/reference/ENVIRONMENT.md`). The per-workload API keys aren't here at all: see [gateway → API keys](gateway.md#api-keys).
+
+## Host-resolved values
+
+These are resolved on the server itself. Set them by hand only if resolution fails:
+
+- `SEARXNG_LAN_IP`, from `search.home`: see [gateway → Web search](gateway.md#web-search).
+- `HOST_VIDEO_GID` / `HOST_RENDER_GID`, from the host's `video` / `render` groups: see [architecture → GPU access](architecture.md#gpu-access).
+
+## Gum
+
+The conflict screen uses [gum](https://github.com/charmbracelet/gum), a single static binary. It's vendored as a release tarball under `scripts/vendor/`, because the server has no outbound internet access. Other architectures fall back to downloading it, and if that fails, to a plain-bash prompt. The extracted binary is cached under `.cache/gum/` (gitignored).
+
+## Model downloads
+
+One `downloader` service in the `tools` compose profile fetches every file listed in [`models.yml`](../models.yml) (`file` + `url` per entry). `update.sh` runs it every time. Files that already exist are skipped, so re-runs don't re-download. To add a model, add an entry there; to load it, point the matching `LLAMA_*_FILE` key in `.env` at the same `file` name. The two are not linked, so a typo means llama-server can't find the file.
+
+- **Partial downloads** go to `<file>.part` and are only renamed once complete. Otherwise an interrupted download would leave a truncated file that the skip check treats as done.
+- **Files live in the named `models` volume**, not a host bind-mount. The downloader runs as root because the named volume is created root-owned.
+- **It needs internet on each run**: it installs `curl` and `yq` from Alpine's package mirror before reading the list.
+
+### Forcing a model re-download
+
+The skip check means `update.sh` never picks up an upstream change to a file that's already on disk (for example, a re-uploaded GGUF with the same name). [`scripts/redownload-model.sh`](../scripts/redownload-model.sh) forces it: with no argument it refreshes `LLAMA_MODEL_FILE` from `.env`, or pass a `file` name from `models.yml` (for example the drafter). It reuses the compose downloader, so the URL still lives only in `models.yml`.
+
+- The old file is renamed to `<file>.old` and deleted only after the new download succeeds. If the download fails or is interrupted, the old file is put back. This needs room for both copies in the `models` volume at once.
+- It ends by restarting `llama-server`, because the running process keeps the old file mapped until it restarts.
+
+## Speed bench
+
+`speed-bench` runs llama.cpp's [SPEED-Bench client](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/bench/speed-bench/README.md) against the running `llama-server`. It measures decode speed (`avg_pred_t/s`), latency and draft acceptance per category. Use it to compare drafters and their flags with numbers instead of gut feel. It's in the `tools` profile, so `update.sh` never builds or runs it. The first run builds the image.
+
+- **The scripts are pinned** to one llama.cpp commit (`LLAMA_CPP_REF` in the Dockerfile). The server image is not pinned, so bump the ref if the client stops matching the server's response format.
+- **It reaches the backend over the compose network**, so llama-server needs no host port. The dataset (`nvidia/SPEED-Bench`, not gated) downloads on first run and is cached in the `hf-cache` volume. Results land in `bench-results/` (gitignored).
+- **It measures speed, not correctness.** The output text isn't saved, so it won't catch a drafter that corrupts tokens.
+
+Run it on the server with `./scripts/bench.sh`. It wraps the compose calls and creates `bench-results/` first, so Docker doesn't create it root-owned. Extra arguments go to the SPEED-Bench client. Pass `--concurrency` equal to `LLAMA_PARALLEL` for throughput, or leave it at 1 for the single-session speed one CLI sees:
+
+```sh
+./scripts/bench.sh run spec --category coding --limit 8
+```
+
+To get a speedup figure, also run a **baseline** without speculative decoding. Temporarily remove the `--spec-*` and `-md` lines from the llama-server command, run `docker compose up -d llama-server`, then run `./scripts/bench.sh run baseline` with the same arguments. Keep `--bench`, `--category`, `--osl` and `--limit` identical. Restore the lines with `git checkout docker-compose.yml` and `docker compose up -d llama-server`, then compare:
+
+```sh
+./scripts/bench.sh compare baseline spec
+```

docs/proxy-key-onboarding.md

@@ -1,26 +0,0 @@
-# Onboarding a workload onto the AI gateway
-
-How to issue a new per-workload API key against the OmniRoute gateway (see [issue #31](https://git.arthurerlich.de/haylan/LLM-Server/issues/31) — the LiteLLM → OmniRoute migration; original gateway rationale in [issue #10](https://git.arthurerlich.de/haylan/LLM-Server/issues/10) / `docs/research/proxy-tool-choice.md`), so a new workload (a code-reviewer tool, Paperless-OCR, Gitea code review, etc.) gets its own key and its own visible usage/spend.
-
-No workload in this stack itself needs a key right now — every client is external (a coding CLI, or another self-hosted service). There's no scripted mint yet either way: `POST /api/keys` needs a dashboard login session (`ManagementSessionAuth`), not a static bearer key like LiteLLM's old `/key/generate`, and that flow hasn't been verified against a live instance (see [issue #37](https://git.arthurerlich.de/haylan/LLM-Server/issues/37)). Create every key by hand for now, via the dashboard steps below.
-
-## Create the key
-
-1. Log into the omniroute dashboard. `DASHBOARD_PORT` (20128) is never published to the host (see `docker-compose.yml`'s `omniroute` service) — from the R9700 box itself, find the container's own address (`docker inspect -f '{{.NetworkSettings.Networks.ai_stack.IPAddress}}' omniroute`) and browse to `http://<that-ip>:20128` (the host can reach a container's bridge-network IP directly, published port or not). From elsewhere, SSH port-forward instead: `ssh -L 20128:<container-ip>:20128 <host>`, then browse `http://localhost:20128`.
-2. "Keys" → "Create API key".
-3. Label it `<workload>-<purpose>` — a short slug matching the workload, e.g. `paperless-ocr`, `gitea-code-review`, `claude-code-cli`. This label is the ledger: the dashboard lists keys by label, so there's no separate tracking doc to keep in sync.
-4. Copy the key value shown — it's only shown once at creation, per OmniRoute's docs.
-
-Once `POST /api/keys`'s session-auth flow is worked out (issue #37), the equivalent `curl` here can replace this manual step, the way `update.sh` used to automate LiteLLM's `/key/generate`.
-
-## Hand it to the workload
-
-Drop the key into that workload's own `.env` (or equivalent config) — never into this repo. Each workload's config is the source of truth for its own credential, same pattern as `.env.example` in this repo for the existing stack.
-
-## Retiring or rotating a key
-
-No scheduled rotation. Revoke the key by hand in the dashboard ("Keys" → delete) only when:
-- the workload is retired, or
-- the key is suspected leaked/compromised.
-
-Then remove it from that workload's `.env`.

docs/research/README.md

@@ -0,0 +1,37 @@
+# Research index
+
+[← back to README](../../README.md)
+
+Each file here is a **dated snapshot**: research or a decision record as it stood when it was written. The files are never edited afterwards, and their status lives only in this table. A *superseded* doc is kept as history. Read it for *why* a choice was made, never for how the stack works today (for that, see [`architecture.md`](../architecture.md) and the source it links).
+
+| Doc | Status | Note |
+|---|---|---|
+| [500k-context-2-parallel-agents-gpu-roadmap.md](500k-context-2-parallel-agents-gpu-roadmap.md) | Background research | Hardware exploration for larger contexts. Not an active plan, and neither is [`../server-planing.md`](../server-planing.md), whatever this doc says about it |
+| [classifier-context-window-scaling.md](classifier-context-window-scaling.md) | Current | Classifier model sizing decision |
+| [colibri-ai-stack-evaluation.md](colibri-ai-stack-evaluation.md) | Current | Rejected option ("watch, don't integrate"). It lists ComfyUI, which has since been removed |
+| [connecting-cloud-providers.md](connecting-cloud-providers.md) | Superseded by the OmniRoute migration | Written for LiteLLM. It mentions Open WebUI, which has been removed |
+| [dashscope-websearch-selfhost-options.md](dashscope-websearch-selfhost-options.md) | Current | Why DashScope web search isn't emulated |
+| [dual-r9700-speed-and-offload.md](dual-r9700-speed-and-offload.md) | Current | Second R9700: split modes, 27B Q8_K_XL vs Flash-Next, RAM/SSD offload, Strata. Proposals only, nothing applied |
+| [fast-model-choice.md](fast-model-choice.md) | Current (drift) | Classifier model choice. Its addendum records the service name (`qwen-classifier`) and quant drift |
+| [langchain-pgvector-vs-litellm-pgvector.md](langchain-pgvector-vs-litellm-pgvector.md) | Superseded by the OmniRoute migration | LiteLLM-era RAG comparison |
+| [lazytainer-omniroute-idle-stop.md](lazytainer-omniroute-idle-stop.md) | Superseded: Lazytainer and `switch-model.sh` removed | Root cause of why Lazytainer's idle-stop never fired behind the gateway, which is the reason it was removed |
+| [litellm-knowledgebase.md](litellm-knowledgebase.md) | Superseded by the OmniRoute migration | LiteLLM vector-store feature. Its claim that Open WebUI is qdrant's only user no longer holds |
+| [litellm-searxng-search.md](litellm-searxng-search.md) | Superseded by [omniroute-qwen-websearch.md](omniroute-qwen-websearch.md) | Still the origin of the `search.home` DNS gotcha ([gateway → Web search](../gateway.md#web-search)) |
+| [omniroute-account-semaphore-timeout.md](omniroute-account-semaphore-timeout.md) | Current | Gateway timeout: per-connection semaphore |
+| [omniroute-direct-response-timeout-outage-2026-09-15.md](omniroute-direct-response-timeout-outage-2026-09-15.md) | Current | Gateway timeout: direct response headers, plus the ROCm stall incident |
+| [omniroute-memory-tool-injection-breaks-classifier.md](omniroute-memory-tool-injection-breaks-classifier.md) | Current | Memory tools must stay off for the classifier connection |
+| [omniroute-non-ping-sse-stream-timeout.md](omniroute-non-ping-sse-stream-timeout.md) | Current | Gateway timeout: first non-ping SSE event, and why `--cache-reuse` is set |
+| [omniroute-qwen-websearch.md](omniroute-qwen-websearch.md) | Current | Qwen Code routing and web search through the gateway |
+| [opencode-auto-compact.md](opencode-auto-compact.md) | Current | OpenCode's client-side compaction |
+| [opencode-cli-setup.md](opencode-cli-setup.md) | Superseded (v1-era) | Written for OpenCode v1. Its config shape (`provider`, `npm`, `options`) is deprecated, and its base URL points at a backend directly. Use the gateway and the v2 config ([setup](../coding-cli-setup/opencode.md)) |
+| [ponytail-audit-2026-09-09.md](ponytail-audit-2026-09-09.md) | Superseded (point-in-time audit) | Findings are fixed or folded into the docs cleanup |
+| [proxy-request-priority.md](proxy-request-priority.md) | Superseded by the OmniRoute migration | Priority tiers via LiteLLM's scheduler, never built on OmniRoute |
+| [proxy-shadow-pricing.md](proxy-shadow-pricing.md) | Superseded by the OmniRoute migration | Shadow cost was wired into LiteLLM config, and nothing implements it now |
+| [proxy-tool-choice.md](proxy-tool-choice.md) | Superseded by the OmniRoute migration | Chose LiteLLM. It mentions Open WebUI, which has been removed |
+| [inference-engine-spec-decoding-multi-gpu.md](inference-engine-spec-decoding-multi-gpu.md) | Current | llama.cpp vs. vLLM vs. SGLang on R9700: drafters, multi-GPU, licenses, `--cache-ram` misspelling bug. Not benchmarked |
+| [qwen3.8-27b-decode-speed.md](qwen3.8-27b-decode-speed.md) | Current | Decode tokens/s: why DFlash2 acceptance and speed fall with depth (draft length is 5, not 7), n_max/p_min, KV and slot costs, bandwidth ceiling, ordered experiment plan. Builds on the throughput-tuning note. Proposals only, nothing applied |
+| [qwen3.8-27b-quant.md](qwen3.8-27b-quant.md) | Current | Main model quant and VRAM budget |
+| [qwen3.8-27b-throughput-tuning.md](qwen3.8-27b-throughput-tuning.md) | Current | Tokens/s tuning from the server log, DDR RAM as a cache tier, comparison with five outside R9700 configs. Proposals only, nothing applied |
+| [qwen3.8-27b-tool-calling.md](qwen3.8-27b-tool-calling.md) | Current | Tool-calling parser risk for the main model |
+| [rocm-gpu-pin-and-render-group.md](rocm-gpu-pin-and-render-group.md) | Current (name drift) | Findings apply to `llama-server` + `qwen-classifier`. The service names in it are older |
+| [voidllm-evaluation.md](voidllm-evaluation.md) | Superseded by the OmniRoute migration | VoidLLM vs. LiteLLM, both moot |

docs/research/classifier-context-window-scaling.md

@@ -0,0 +1,249 @@
+# Does the qwen-classifier need to match the main model's context window, and would upgrading it to Qwen3-8B or Qwen3-Coder-30B-A3B-Instruct fit on the current R9700?
+
+**Date:** 2026-09-15
+**Question raised:** should `qwen-classifier` be resized to `LLAMA_CTX_SIZE / LLAMA_PARALLEL` (131072, matching the
+main model's per-slot context), should the classifier model itself move up to Qwen3-8B or
+Qwen3-Coder-30B-A3B-Instruct, and does that need a second GPU?
+
+**Answer: No, no, and not for this reason.** qwen-code's own docs state no context-size requirement for
+`fastModel` at all — the "match the main model" premise doesn't come from any primary source. Separately, and
+independently of context size: neither Qwen3-8B nor Qwen3-Coder-30B-A3B-Instruct fits in the ~6.1GiB of VRAM
+actually free on the card today, at *any* context length, once weights alone are counted — this is a raw-VRAM
+problem, not a context-window problem, exactly matching the "qwen8b needs to offload more to the RAM" intuition
+in the request. A second GPU would solve the VRAM problem (and incidentally remove this repo's own
+`GPU_MAX_HW_QUEUES=1` ROCm#5706 workaround from applying to this pair), but isn't deployed hardware today —
+it's a rack-build/acquisition question, not a config change.
+
+## 1. Does qwen-code require the fast/classifier model to match the main model's context window?
+
+No — checked against the user's own three linked pages, fetched directly:
+
+- **`fastModel` settings docs**: "Model used for generating prompt suggestions and speculative execution,"
+  configurable via `inherit` (main model), `fast`, a model ID, or `authType:model-id`; "Leave empty to use the
+  main model." The docs recommend "a smaller/faster model (e.g., `qwen3-coder-flash`) reduces latency and
+  cost" — **no context-window size or capacity requirement is stated anywhere on this page.**
+  Source: [qwen-code docs — Configuration / Settings, `#fastmodel`](https://qwenlm.github.io/qwen-code-docs/en/users/configuration/settings/#fastmodel)
+
+- **Approval Mode / Auto Mode classifier docs**: describes what the classifier evaluates (shell commands,
+  network calls, out-of-workspace edits) and its allow/block behavior, but **does not name a specific model or
+  state any context-window requirement** — the only operational note is that "when the classifier API is
+  unreachable, the action is blocked rather than allowed."
+  Source: [qwen-code docs — Approval Mode, `#4-auto-mode---classifier-driven-approval`](https://qwenlm.github.io/qwen-code-docs/en/users/features/approval-mode/#4-auto-mode---classifier-driven-approval)
+
+- **Auto Mode "How it works"**: confirms the two-stage design (Stage 1: ~300ms, `{shouldBlock}` only; Stage 2:
+  chain-of-thought reconsideration, only on a Stage-1 block) and what data reaches the classifier — user text,
+  assistant tool-use calls, and tool-specific projections (truncated edit content, fetch URLs, shell command
+  text). **Tool results are explicitly never sent to the classifier.** It "uses your configured fast model
+  (`/model --fast`)," falling back to the main session model only if none is set. **No statement anywhere
+  requires or implies the fast model's context window match the main model's.**
+  Source: [qwen-code docs — Auto Mode, `#how-it-works`](https://qwenlm.github.io/qwen-code-docs/en/users/features/auto-mode/#how-it-works)
+
+This confirms and sharpens what this repo's own `fast-model-choice.md` already found by reading qwen-code's
+source directly (`packages/core/src/permissions/classifier-transcript.ts`: `MAX_TRANSCRIPT_MESSAGES=40`,
+`MAX_HISTORICAL_ACTION_CHARS=4000`/message, worst case ~40-50K tokens, live-tested at 15,116 prompt tokens) —
+that doc already called the original `131072` in `settings.json` "copied from the main model's entry, not a
+real qwen-code requirement." The three docs pages fetched here add nothing that contradicts that: **there is no
+primary-source basis for `LLAMA_CTX_SIZE / LLAMA_PARALLEL` symmetry between the two models.** The current
+`65536` classifier ctx already carries ~1.5x margin over the real worst case.
+
+## 2. VRAM math for Qwen3-8B and Qwen3-Coder-30B-A3B-Instruct as classifier candidates
+
+Same method this repo already uses (`qwen3.8-27b-quant.md`, `fast-model-choice.md` §5): per-token KV cache =
+`layers × 2(K+V) × kv_heads × head_dim × bytes`, read directly from each model's own `config.json`.
+
+### Qwen3-8B
+
+- Architecture (`Qwen/Qwen3-8B` `config.json`): `num_hidden_layers: 36`, `num_key_value_heads: 8`,
+  `num_attention_heads: 32`, `head_dim: 128`, `hidden_size: 4096`, `max_position_embeddings: 40960`,
+  `rope_scaling: null`.
+  Source: [Qwen/Qwen3-8B `config.json`](https://huggingface.co/Qwen/Qwen3-8B/raw/main/config.json)
+- **Native context is 32,768 tokens**, not the 262,144 the user's brief assumed (that number belongs to
+  Qwen3-4B-Instruct-2507, a different, non-reasoning 2507-refresh model — Qwen3-8B is the earlier,
+  thinking-capable Qwen3 architecture with a materially smaller native window). Extending past 32K needs YaRN:
+  > "Qwen3 natively supports context lengths of up to 32,768 tokens. For conversations where the total length
+  > (including both input and output) significantly exceeds this limit, we recommend using RoPE scaling
+  > techniques to handle long texts effectively."
+  and llama.cpp-specific YaRN invocation is given explicitly:
+  `./llama-cli ... -c 131072 --rope-scaling yarn --rope-scale 4 --yarn-orig-ctx 32768`, with a documented
+  caveat that "all the notable open-source frameworks implement **static** YaRN, which means the scaling
+  factor remains constant regardless of input length, potentially impacting performance on shorter texts."
+  Source: [Qwen/Qwen3-8B-GGUF — Processing Long Texts](https://huggingface.co/Qwen/Qwen3-8B-GGUF#processing-long-texts)
+- **Weights** (official Qwen quants, fetched from the GGUF repo file list): Q5_K_M = 5.85 GB, Q8_0 = 8.71 GB.
+  Source: [Qwen/Qwen3-8B-GGUF](https://huggingface.co/Qwen/Qwen3-8B-GGUF)
+  *(Not independently verified: unsloth's equivalent `UD-Q4_K_XL` quant, which is what this repo's
+  `docker-compose.yml`/`.env.example` actually download for every model deployed so far — the unsloth file
+  size wasn't fetched, only the official Qwen quants above. Treat Q5_K_M/Q8_0 as a reasonable bound, not the
+  exact file this repo would pull.)*
+- Per-token KV cache: `36 × 2 × 8 × 128 × 2 bytes = 144 KiB/token` fp16 — identical to Qwen3-4B-Instruct-2507's
+  figure in `fast-model-choice.md` §5, since both share the same `layers/kv_heads/head_dim` triple.
+
+| Context | KV (fp16) | KV (q8_0) | KV (q4_0, current classifier setting) |
+|---|---|---|---|
+| 65,536 (current classifier ctx) | 9.0 GiB | 4.5 GiB | **2.25 GiB** |
+| 131,072 (user's proposed "match main model") | 18.0 GiB | 9.0 GiB | **4.5 GiB** |
+
+Weights + KV (q4_0, smallest realistic combo):
+
+| Context | Q5_K_M weights + q4_0 KV | Q8_0 weights + q4_0 KV |
+|---|---|---|
+| 65,536 | 5.85 + 2.25 = **8.1 GB** | 8.71 + 2.25 = **10.96 GB** |
+| 131,072 | 5.85 + 4.5 = **10.35 GB** | 8.71 + 4.5 = **13.21 GB** |
+
+### Qwen3-Coder-30B-A3B-Instruct
+
+- Architecture (fetched from the shared Qwen3-30B-A3B-family `config.json`): `num_hidden_layers: 48`,
+  `num_key_value_heads: 4`, `num_attention_heads: 32`, `head_dim: 128`, `hidden_size: 2048`, **MoE**:
+  `num_experts: 128`, `num_experts_per_tok: 8` (8 of 128 experts active per token — confirms this is a sparse
+  MoE model, not a dense one like the 27B or 8B candidates; the "active params" figure describes *compute*
+  per token, not memory footprint — **all 128 experts' weights still have to be resident** wherever the model
+  is loaded, GPU or RAM).
+  Source: [Qwen/Qwen3-30B-A3B-family `config.json`](https://huggingface.co/Qwen/Qwen3-30B-A3B/raw/main/config.json)
+- **UD-Q4_K_XL file size (the exact quant/quantizer this repo already standardizes on): 17.7 GB.** 30.5B total /
+  3.3B activated parameters. Native context "262,144 natively... can be extended further using Yarn to reach
+  1M tokens."
+  Source: [unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF](https://huggingface.co/unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF?show_file_info=Qwen3-Coder-30B-A3B-Instruct-UD-Q4_K_XL.gguf)
+- Per-token KV cache: `48 × 2 × 4 × 128 × 2 bytes = 96 KiB/token` fp16 (smaller per-token than the 8B/4B
+  candidates, since `num_key_value_heads` is 4 here vs. 8 — but this saving is irrelevant given the weights
+  size below).
+
+| Context | KV (fp16) | KV (q8_0) | KV (q4_0) |
+|---|---|---|---|
+| 65,536 | 6.0 GiB | 3.0 GiB | **1.5 GiB** |
+| 131,072 | 12.0 GiB | 6.0 GiB | **3.0 GiB** |
+
+Weights + KV (q4_0):
+
+| Context | Total |
+|---|---|
+| 65,536 | 17.7 + 1.5 = **19.2 GB** |
+| 131,072 | 17.7 + 3.0 = **20.7 GB** |
+
+## 3. Does either candidate fit the ~6.1 GiB actually free on the card today?
+
+**No — neither does, at either context size, even at the smallest quant/KV-quant combination tested.**
+
+- Qwen3-8B's cheapest realistic combination (Q5_K_M weights + q4_0 KV at the *current* 65536 ctx, not even
+  the proposed 131072) is **8.1 GB — already ~2 GB over the measured 6.1 GiB free budget**, before accounting
+  for compute-buffer/batch overhead that `fast-model-choice.md` §"Implementation note" already found could add
+  meaningfully on top of the naive weights+KV estimate (that's exactly why the 4B classifier ended up needing
+  `--flash-attn on` and partial 28/36-layer offload instead of the originally-predicted comfortable full-GPU
+  fit).
+- Qwen3-Coder-30B-A3B-Instruct isn't close at any setting tested — its weights alone (17.7 GB) are triple the
+  entire free budget, and this doesn't change with context size since the weights term dominates.
+- This is a **VRAM-capacity problem, not a context-window problem** — directly confirming the "qwen8b needs to
+  offload more to the RAM" intuition in the original request. Reducing context doesn't fix it; the weights
+  don't fit regardless.
+
+**CPU/RAM offload mechanics:** llama.cpp's `--n-gpu-layers` is documented as "max. number of layers to store in
+VRAM, either an exact number, `'auto'`, or `'all'`" — the layers not selected are computed on CPU, with the
+model loaded via mmap by default (same mechanism this repo's own `.env.example` already documents for
+`LLAMA_GPU_LAYERS`: "if GPU+RAM ever can't hold the working set, the OS pages the rest in from disk
+automatically"). For the MoE Coder-30B-A3B model specifically, this repo's own `.env.example` already flags the
+more targeted alternative — `--n-cpu-moe`/`--cpu-moe`/`--override-tensor "exps"` — as the flags that "target
+Mixture-of-Experts models (e.g. Qwen3.8-2.4T-A95B)," i.e. exactly this model's architecture: these offload only
+expert-tensor weights to CPU while keeping attention/shared layers and KV cache on GPU, which is the
+mechanically correct lever for an MoE model, unlike the blunt `--n-gpu-layers` used for the dense 8B/27B/4B
+models. **Neither llama.cpp's own README nor the Qwen model cards fetched here document a quantified
+performance cost for partial offload** — no primary source gives a "N layers offloaded = X% slower" figure.
+Source: [llama.cpp `tools/server/README.md`](https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/server/README.md)
+
+What *is* directly measured, in this repo's own deployment history: CPU-only was tried first for the current,
+much smaller 4B classifier and rejected — "too slow in practice: real classification calls blew past
+OmniRoute's 60s timeout and retry-looped (504→499→504)" (`docker-compose.yml`'s `qwen-classifier` comment
+block). An 8B dense model has roughly double the compute of the 4B model per token; a 30B-A3B model's *routing*
+overhead on CPU (choosing 8 of 128 experts per token, each a separate weight lookup) adds a different kind of
+cost that neither this repo nor the sources fetched here have measured. **Given the classifier's Stage 1 has an
+explicit ~300ms latency budget** (`fast-model-choice.md` §1, from qwen-code's own docs), and this repo already
+has one concrete data point that CPU offload breaks that budget at a smaller model size, extending either
+candidate onto significant CPU offload carries real, unquantified latency risk — the same failure mode already
+observed once, at a favorable (smaller) model size.
+
+## 4. Does a second GPU solve this, and is one actually available?
+
+**Not today.** `docs/server-planing.md` is a rack-build plan for "3-4x AMD Radeon AI PRO R9700 (32GB) GPUs" —
+a future-state document, not present inventory. Every GPU-facing comment in this repo's own
+`docker-compose.yml`/`.env.example`/`rocm-gpu-pin-and-render-group.md` consistently refers to "the single 32GB
+R9700" and measures the "~6.1GiB free" budget against one physical card holding both `llama-server` and
+`qwen-classifier`. Adding a second GPU is a hardware-acquisition and rack-build question — physically sourcing,
+installing, and power/PCIe-provisioning a card per `server-planing.md`'s own build plan — not a
+`docker-compose.yml`/`.env.example` change.
+
+**If a second GPU were added**, it would directly remove one already-documented risk for this specific pair:
+this repo's own `rocm-gpu-pin-and-render-group.md` traced the GPU-pinned-at-100%/ROCm#5706 bug to its precise
+trigger condition —
+
+> "The pin only appears with two concurrent HIP-context-holding processes **on the same GPU**... Root cause: an
+> AMD MES (Micro Engine Scheduler) firmware bug triggered by HIP hardware-queue creation."
+Source: [ROCm/ROCm#5706](https://github.com/ROCm/ROCm/issues/5706), via this repo's own
+[`rocm-gpu-pin-and-render-group.md`](rocm-gpu-pin-and-render-group.md)
+
+Since the confirmed trigger is *two HIP contexts sharing one physical card*, moving the classifier to its own,
+second GPU would put each service on a single-HIP-context card — the condition that trips the bug wouldn't
+exist for this pair anymore, and the `GPU_MAX_HW_QUEUES=1` workaround currently applied to both services
+specifically because they share one card would no longer be load-bearing for *this* pair (it would still apply
+if any future third service shared a card with either model). This wasn't independently re-verified across two
+*separate* physical cards by any source fetched in this pass — it's a direct extrapolation from the confirmed
+root cause, same category of caveat that doc's own author already flagged for its within-one-card claim.
+
+## Bottom line / recommendation
+
+1. **Don't apply `LLAMA_CTX_SIZE / LLAMA_PARALLEL` symmetry to the classifier.** No qwen-code primary source
+   states or implies the fast/classifier model needs to match the main model's context window. The real
+   requirement (§1, already established in `fast-model-choice.md`) is ~40-50K tokens worst case; the current
+   `65536` already has margin. Doubling to 131072 would only double VRAM spent on KV cache for a model that
+   won't otherwise fit anyway (§2-3).
+2. **Don't upgrade the classifier to Qwen3-8B or Qwen3-Coder-30B-A3B-Instruct on the current single-GPU setup.**
+   Neither fits the ~6.1GiB actually free, at any context size — this is a weights-size problem, not a
+   context-window problem. Forcing it would mean either (a) shrinking the main 27B model's own VRAM footprint
+   to make room (a real trade-off against the primary model, not evaluated here), or (b) CPU/partial offload,
+   which this repo has direct, measured evidence already breaks the classifier's latency budget at a *smaller*
+   model size than either candidate.
+3. **A second GPU is the clean fix for VRAM contention and would also retire the ROCm#5706 workaround's
+   relevance for this pair — but it isn't deployed hardware today.** `server-planing.md` is a future build
+   plan; this is an acquisition/rack-build decision, not something achievable via a config change right now.
+4. If the actual underlying motivation is classifier *quality* (not context capacity), that's a separate,
+   legitimate question this doc doesn't answer — worth its own research pass rather than solving it via a
+   bigger model that doesn't fit the hardware.
+
+## Sources
+
+- [qwen-code docs — Configuration / Settings, `#fastmodel`](https://qwenlm.github.io/qwen-code-docs/en/users/configuration/settings/#fastmodel)
+- [qwen-code docs — Approval Mode, `#4-auto-mode---classifier-driven-approval`](https://qwenlm.github.io/qwen-code-docs/en/users/features/approval-mode/#4-auto-mode---classifier-driven-approval)
+- [qwen-code docs — Auto Mode, `#how-it-works`](https://qwenlm.github.io/qwen-code-docs/en/users/features/auto-mode/#how-it-works)
+- [Qwen/Qwen3-8B `config.json`](https://huggingface.co/Qwen/Qwen3-8B/raw/main/config.json)
+- [Qwen/Qwen3-8B-GGUF](https://huggingface.co/Qwen/Qwen3-8B-GGUF)
+- [Qwen/Qwen3-8B-GGUF — Processing Long Texts](https://huggingface.co/Qwen/Qwen3-8B-GGUF#processing-long-texts)
+- [Qwen/Qwen3-30B-A3B-family `config.json`](https://huggingface.co/Qwen/Qwen3-30B-A3B/raw/main/config.json)
+- [unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF](https://huggingface.co/unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF?show_file_info=Qwen3-Coder-30B-A3B-Instruct-UD-Q4_K_XL.gguf)
+- [llama.cpp `tools/server/README.md`](https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/server/README.md)
+- [ROCm/ROCm#5706](https://github.com/ROCm/ROCm/issues/5706)
+- [docs/research/fast-model-choice.md](fast-model-choice.md) (this repo — classifier transcript sizing,
+  qwen-classifier's real deployment history)
+- [docs/research/qwen3.8-27b-quant.md](qwen3.8-27b-quant.md) (this repo — KV-cache-from-config.json method
+  reused here)
+- [docs/research/rocm-gpu-pin-and-render-group.md](rocm-gpu-pin-and-render-group.md) (this repo — ROCm#5706
+  trigger condition and `GPU_MAX_HW_QUEUES` scoping)
+- [docs/server-planing.md](../server-planing.md) (this repo — confirms only 1 of a planned 4 GPUs is deployed)
+- `docker-compose.yml`, `.env.example` (this repo — current `qwen-classifier`/`llama-server` config and the
+  measured "~6.1GiB free" VRAM figure)
+
+## Confidence / uncertainty summary
+
+- **High confidence:** qwen-code's `fastModel`/Auto-Mode docs state no context-window requirement (direct
+  quotes from all three linked pages); Qwen3-8B's native 32,768 context and YaRN caveat (direct model-card
+  quote); Qwen3-Coder-30B-A3B-Instruct's MoE architecture and 17.7GB Q4_K_XL file size (direct from the
+  quantizer's own repo page); the KV-cache-per-token math for both candidates (computed directly from each
+  model's own `config.json`, same method already validated in this repo's prior research); the ROCm#5706
+  trigger condition being scoped to two HIP contexts on the *same* GPU (direct quote from this repo's own
+  prior research, itself sourced from the upstream issue).
+- **Medium confidence:** the exact unsloth `UD-Q4_K_XL`-equivalent file size for Qwen3-8B — only the official
+  Qwen quants (Q5_K_M/Q8_0) were fetched, not unsloth's own repo, so the real number this repo would actually
+  download wasn't directly verified (bounded reasonably by the Q5_K_M figure, which is already the smallest
+  realistic option and still doesn't fit). The claim that CPU/partial-offload latency risk scales unfavorably
+  for larger/MoE models is a reasoned extrapolation from this repo's one measured data point (4B CPU-only
+  rejected) plus general MoE-routing-overhead reasoning, not a directly measured benchmark for either candidate.
+- **Low confidence / not independently verified:** whether `GPU_MAX_HW_QUEUES=1`/ROCm#5706 genuinely has zero
+  relevance across two *separate* physical GPUs (extrapolated from the confirmed same-GPU trigger condition,
+  same caveat this repo's own prior research already flagged for its own claim); no primary source found that
+  quantifies llama.cpp's actual inference-speed penalty for partial `--n-gpu-layers` or `--n-cpu-moe` offload
+  in general — this is a documented gap in the sources checked, not a guessed number.

docs/research/dual-r9700-speed-and-offload.md

@@ -0,0 +1,240 @@
+# Research: fastest inference on 2x R9700 (27B Q8_K_XL vs Flash-Next, multi-GPU modes, RAM/SSD offload, Strata)
+
+**Date:** 2026-10-29
+
+**Question:** A second Radeon AI PRO R9700 arrives in about a week. What gives the fastest decode (and prefill) for agentic coding on this box: `Qwen3.8-27B-UD-Q8_K_XL`, `Qwen3.8-Flash-Next-UD-Q4_K_XL`, or something else? Which llama.cpp multi-GPU mode helps, does RAM/SSD offload work with 32 GB of RAM, and is Strata worth switching to?
+
+**Scope note:** research only. Nothing in `docker-compose.yml` or `.env.example` was changed, so nothing needs deploying.
+
+**Hardware assumed:** Ryzen 9 5950X, 32 GB DDR4 dual channel (reported ~94 % used, swap in use), ASRock B550M Pro4, 1 TB NVMe, Ubuntu 24.04, kernel 6.8, 1x R9700 now and 2x soon.
+
+## TL;DR
+
+| # | Finding | Confidence |
+|---|---|---|
+| 1 | **The second card does not make the dense 27B decode faster with `-sm layer`.** Layer split runs the cards one after the other, so one token still reads every weight at one card's bandwidth. It only adds VRAM. | High (llama.cpp docs) |
+| 2 | **`UD-Q8_K_XL` is a quality move, not a speed move.** It reads ~30.1 GB per token against ~16.8 GB for today's `UD-Q4_K_XL`. Estimated decode is ~15 t/s without speculation and ~25 t/s with DFlash2, against ~46 t/s measured today. | Medium (estimate, math below) |
+| 3 | **`-sm tensor` is the only mode that can speed up one stream, and on ROCm it is immature.** The one measured 2-card ROCm result for this exact model is 1.31x one card (2x 7900 XTX). DFlash2 crashes with `-sm tensor` (open bug), and MTP plus `-sm tensor` has lockup reports. Your second slot is PCIe 3.0 x4 behind the chipset, which hurts the per-layer all-reduces. | Medium |
+| 4 | **Best use of the second card for 2 parallel agents: two independent single-GPU `llama-server` replicas**, one per card, each `--parallel 1`. Each agent keeps today's ~46 t/s, the decode drop when the other slot prefills goes away, and each slot gets the full 262144 tokens instead of 131072. Needs a second backend connection in OmniRoute with per-conversation stickiness. | Medium-high |
+| 5 | **Flash-Next UD-Q4_K_XL (111.3 GB) does not fit.** Even with the 28.8 GB n-gram table left on SSD, 82.5 GB of weights remain for 64 GB of VRAM. The ~20-24 GB of experts left over would come from NVMe, because RAM is full. Estimated decode is then ~6-12 t/s. Only the all-in-VRAM quants (`UD-Q2_K_XL`, `UD-IQ3_XXS`) are realistic, at a large quality cost (mean KLD 0.22 / 0.17 vs 0.047). | Medium |
+| 6 | **RAM is the binding constraint, and compose itself is a likely cause of the swap.** `--cache-ram` is set to 24576 MiB (24 GiB) on a 32 GB host that also runs OmniRoute with `mem_limit: 10g`. | High for the config, inferred for the swap |
+| 7 | **Strata:** a real, fast-moving MIT engine built around Flash-Next expert tiering. It has R9700 HIP builds and a layer split for 2 cards, but its design wants all experts in RAM (~88 GB for UD-Q4_K_XL), it does not run the dense 27B at all, and its numbers are self-reported on hosts with 47-503 GB of RAM. Not worth it on 32 GB. Revisit after a RAM upgrade (board max 128 GB). | Medium |
+
+## 1. Platform facts that drive everything
+
+- **R9700:** 32 GB GDDR6, 256-bit, 20 Gbps, **640 GB/s**, PCIe 5.0 x16, 300 W ([Puget Systems listing](https://www.pugetsystems.com/parts/Video-Card/AMD-Radeon-AI-PRO-R9700-32GB-16908/), [Tom's Hardware launch](https://www.tomshardware.com/pc-components/gpus/amd-launches-radeon-ai-pro-r9700-to-challenge-nvidias-ai-market-dominance)). AMD's own product page timed out on fetch, so this is from third-party listings.
+- **Effective bandwidth, calibrated from this repo's data:** no-spec decode for the 27B Q4 on one R9700 is 25-30 t/s across three outside sources ([`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md)). 16.8 GB/token x ~26.4 t/s ≈ **~440 GB/s effective (~69 % of peak)**. All estimates below use 420-450 GB/s.
+- **ASRock B550M Pro4 slots**, from the [ASRock manual (PDF)](https://download.asrock.com/Manual/B550M%20Pro4.pdf) spec page with a Matisse/Vermeer CPU: "2 x PCI Express x16 Slots (PCIE1: Gen4x16 mode; PCIE3: Gen3 x4 mode)", plus one PCIe 3.0 x1 slot (PCIE2). M2_1 is "up to Gen4x4", M2_2 "up to Gen3 x2", and M2_2 shares lanes with SATA3_5_6. Memory is dual channel, max 128 GB. So **the second R9700 runs at PCIe 3.0 x4 (~3.9 GB/s per direction)**.
+- **PCIE3 is behind the chipset.** The manual does not say so explicitly. It follows from the AM4 lane budget: the CPU gives x16 to the GPU, x4 to the CPU-attached M.2, and x4 as the chipset uplink. B550's own lanes and uplink are PCIe 3.0 ([ComputerBase B550 test](https://www.computerbase.de/2020-06/amd-b550-chipsatz-mainboard-test/), [Guru3D](https://www.guru3d.com/review/amd-ryzen-3-3100-and-3300x-review/page-3/)). Card 2 therefore shares the chipset's PCIe 3.0 x4 uplink with M2_2, SATA, USB and LAN. GPU-to-GPU traffic crosses CPU root complex → chipset → slot. Whether ROCm P2P works across that path was not verified. Assume it falls back to staging through host memory.
+- **DDR4:** dual-channel DDR4-3200 peaks at 2 x 8 B x 3200 MT/s = **51.2 GB/s**. Plan for ~35-45 GB/s effective (estimate).
+- **RAM budget today:** `docker-compose.yml` sets `--cache-ram ${LLAMA_CACHE_RAM:-24576}` (24 GiB) and OmniRoute `mem_limit: 10g` (`OMNIROUTE_MEMORY_MB=8192`). That is 34 GiB of permitted use on a 32 GB host before the OS, qdrant and page cache. `--cache-ram` is a ceiling that fills up over time, which fits the reported 94 % plus swap. `docs/architecture.md` already says to "keep it well below the host's free RAM".
+- **Physical and power, unverified:** two 300 W cards plus a 142 W-PPT 5950X need a PSU of roughly 850 W or more. [`500k-context-…roadmap.md`](500k-context-2-parallel-agents-gpu-roadmap.md) already flagged PSU headroom as the limit. A dual-slot card in the bottom slot of a micro-ATX board can also need a fifth rear bracket in the case. Check both before the card arrives.
+
+## 2. Qwen3.8-27B `UD-Q8_K_XL`
+
+**File facts** (HF tree API plus the GGUF header, parsed from the first 12 MB of each file):
+
+| | UD-Q4_K_XL (today) | UD-Q8_K_XL |
+|---|---|---|
+| File size | 17.56 GB | **31.46 GB** |
+| Tensor types | mixed IQ4/Q4_K-Q6_K, Q8_0 for small tensors | Q8_0 FFN/attention, **BF16** `attn_q/k/v` and `output` |
+| `token_embd` (row lookup, not streamed) | 0.72 GB | 1.35 GB |
+| **Bytes streamed per token** | **~16.8 GB** | **~30.1 GB (1.79x)** |
+
+Sources: [HF tree API](https://huggingface.co/api/models/unsloth/Qwen3.8-27B-GGUF/tree/main), [file page](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF?show_file_info=Qwen3.8-27B-UD-Q8_K_XL.gguf). Both files carry the built-in MTP head (`qwen35.block_count = 65`, `nextn_predict_layers = 1`).
+
+**Architecture** ([unsloth config.json](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/raw/main/config.json)): dense, `Qwen3_5ForConditionalGeneration`, 64 layers, hybrid. 48 Gated DeltaNet (`linear_attention`) layers and 16 `full_attention` layers (`full_attention_interval: 4`), 24 Q heads, 4 KV heads, head_dim 256, 262144 native context. Every weight is read for every token.
+
+**Fit on 2x32 GB.** KV is 16 layers x 2 x 4 heads x 256 x 1.0625 B (q8_0) ≈ 34.8 KB/token, so ~9.1 GB at 262144 tokens. 31.5 GB weights + 1.1 GB DFlash2 Q4_K_M ([z-lab GGUF tree](https://huggingface.co/api/models/z-lab/Qwen3.8-27B-DFlash2-GGUF/tree/main)) + 9.1 GB KV + ~3-5 GB compute buffers per card comes to **~46-50 GB of 64 GB** (estimate). It fits with room. It does not fit on one card.
+
+**Decode estimate** (bandwidth-bound, ~440 GB/s per card):
+
+| Config | Math | Est. decode, no spec | With speculation (x1.7, the ratio this repo measures for DFlash2: 45.9 / ~27) |
+|---|---|---|---|
+| Q4_K_XL, 1 card (today) | 16.8 / 440 = 38 ms | ~26 t/s (measured 25-30) | **45.9 t/s measured** |
+| Q8_K_XL, `-sm layer` | 30.1 / 440 = 68 ms, cards in series | **~15 t/s** | **~25 t/s** |
+| Q8_K_XL, `-sm tensor` at 1.31x | 15 x 1.31 | ~19 t/s | DFlash2 crashes here (see §3). MTP x1.5 gives ~29 t/s if stable |
+| Q4_K_XL, `-sm tensor` at 1.31x | 26 x 1.31 | ~34 t/s | MTP x1.5 gives ~50 t/s if stable |
+
+All of these except the measured row are estimates. Q8_K_XL roughly halves single-stream speed against today. Pick it only if Q4 quality is shown to be the bottleneck on your own tasks.
+
+## 3. llama.cpp multi-GPU on ROCm
+
+Source: [`docs/multi-gpu.md`](https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md) and `common/arg.cpp` on master (fetched 2026-10-29). `--split-mode` takes `none, layer, row, tensor`.
+
+| Mode | What it does | Decode on bandwidth-bound cards | Status |
+|---|---|---|---|
+| `layer` (default) | Pipeline: each GPU owns a contiguous block of layers and their KV | **No single-stream gain.** One token walks GPU0 then GPU1, each reading only its half, so the total time is the same. Prefill can overlap ubatches across GPUs. The docs say: "priority is a fast prefill. Can tolerate slow interconnect speeds" | Mature |
+| `row` | Old row-split of dense matmuls | Docs: "**Deprecated** … Superseded by `tensor`". [PR #19378](https://github.com/ggml-org/llama.cpp/pull/19378): needs "a synchronization after every single operation", only worth it on old slow GPUs | Avoid |
+| `tensor` | Meta-backend tensor parallelism: weights **and** KV split, all-reduce at sync points | Both cards read their half of each layer at the same time, so it is the only mode with a decode win. Docs: "Performance should be good for multiple NVIDIA GPUs using the CUDA backend, no guarantees otherwise" | **EXPERIMENTAL**. Merged in [#19378](https://github.com/ggml-org/llama.cpp/pull/19378) (2026-04-09) |
+
+**Measured `-sm tensor` numbers that are relevant here:**
+
+- **Qwen3.8-27B Q4_K_XL, 2x RX 7900 XTX, ROCm: 47.06 t/s = 1.31x one card.** Per-card DRAM utilization drops from 65.9 % to 43.0 % when split ([#28863](https://github.com/ggml-org/llama.cpp/issues/28863), closed without a fix). This is the best proxy for a 2x R9700 result.
+- The #19378 author on ROCm (RX 6800 + MI50, RX 9060 XT + MI100): "the performance is bad vs. the `-sm layer` baseline". On 2x RTX 4090, the dense Gemma4 31B decoded **1.36x** faster, while pp2048 fell to 0.61x and the MoE model got slower.
+- [#27825](https://github.com/ggml-org/llama.cpp/pull/27825) (merged 2026-09-15) turned on the internal AllReduce for HIP. On an RX 9070 at x16 plus an RX 6800 XT at **PCIe 4.0 x4**, Gemma4 31B Q6_K: tg 23.62 → 24.15 (+2 %), pp +16 %. RCCL is opt-in at build time (`-DGGML_HIP_RCCL=ON`), "disabled because … it was not universally beneficial" (multi-gpu.md). Whether the official `server-rocm` image builds with RCCL was not checked.
+- 2x R9700 user, Gemma4 26B-A4B Q8_0: `-sm tensor` f16 KV 29.7 t/s vs `-sm layer` 19 t/s ([#23567](https://github.com/ggml-org/llama.cpp/issues/23567)). Same card, a different (MoE) model.
+- A 5950X + dual-R9700 user runs "Qwen 3.6 27B Q8 … with tensor split" and reports trouble mixing tensor-split and layer-split processes on ROCm ([#28582](https://github.com/ggml-org/llama.cpp/issues/28582), open).
+
+**Constraints and bugs** that apply to this stack:
+
+- Architecture support: `qwen35` and `qwen4exp` are **not** on the deny list in `llm_arch_supports_sm_tensor` (`src/llama-arch.cpp`, master). `-sm tensor` was re-enabled for qwen4exp in [#28569](https://github.com/ggml-org/llama.cpp/pull/28569).
+- `-sm tensor` needs flash-attn on (`llama-context.cpp`). Quantized KV is supported since [#23792](https://github.com/ggml-org/llama.cpp/pull/23792) (2026-06-01). llama.cpp's `docs/multi-gpu.md` still says it errors, so the doc is stale there. `--fit` is "not supported with `tensor`", so set `--ctx-size` by hand.
+- **DFlash2 + `-sm tensor` crashes:** `GGML_ASSERT(src_ss[0].axis != GGML_BACKEND_SPLIT_AXIS_0)` ([#28777](https://github.com/ggml-org/llama.cpp/issues/28777), open). The ROCm variant ([#27829](https://github.com/ggml-org/llama.cpp/issues/27829)) was closed as stale, and the fix PR [#27858](https://github.com/ggml-org/llama.cpp/pull/27858) is open. Its tester still hit a different assert, and a commenter says "this is already a problem for ROCm". The workaround in #28777 is `-sm layer`.
+- **MTP + `-sm tensor`:** reproducible lockups on CUDA ([#27122](https://github.com/ggml-org/llama.cpp/issues/27122), closed as stale). One ROCm report (2x V620, Q8_K_XL) saw one hard abort in ~67 h.
+- **gfx1201 flash-attn garbage at some context lengths for Qwen3.8-27B** (head_dim 256): immediate EOS in bands, reproduced on **2x R9700, Linux, `--tensor-split 1.7,1`** ([#30884](https://github.com/ggml-org/llama.cpp/issues/30884), open). Another commenter could not reproduce it through `/v1/chat/completions`. Run a needle test on any new config.
+
+**PCIe x4 chipset slot, estimated effects:**
+
+- `-sm layer` decode: one hidden-state hop per token (5120 x 4 B ≈ 20 KB). Negligible. Prefill: ~10 MB per 512-token ubatch hop, ~3 ms at 3.9 GB/s. Small.
+- `-sm tensor` decode: ~2 all-reduces per layer x 64 layers ≈ 128 small reductions per token. These are latency-bound, not size-bound. At an assumed 20-50 µs each through host memory, that is 2.5-6.4 ms per token, 10-25 % of a 25 ms token (estimate). This is the likely reason #28863 lands at 1.31x and not ~1.9x, and an x4-through-chipset link will not do better.
+- `-sm tensor` prefill: ~2 x tokens x 5120 x 2 B per layer. For a 2048-token ubatch that is ~2.7 GB over 64 layers, ~0.7 s at 3.9 GB/s, so prefill is capped at roughly 3000 t/s plus compute (estimate). Expect prefill to get worse than layer split, as in the #19378 tables.
+
+**Flags:**
+
+- `-ts/--tensor-split` sets proportions. With `layer` the default split follows free memory. With `tensor` the default is an even split.
+- `-mg/--main-gpu` applies only to `-sm none` (pick the card) or `row` (intermediate results). It does nothing for `layer`/`tensor`.
+- Draft placement: `--spec-draft-device/-devd` (default: follows `--device`), `-ngld`, `-otd` (`common/arg.cpp`). Under `layer`, the output head sits on the last GPU. Place the drafter on one card and A/B both cards. No source documents which placement is better for DFlash2, which reads hidden states from several target layers (`n_extract=5`) spread over both cards.
+
+## 4. Qwen3.8-Flash-Next `UD-Q4_K_XL`
+
+**Model facts** ([unsloth model card](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF), GGUF header of shard 1, and shard headers 2-4):
+
+- `general.architecture = qwen4exp`, described as "A Preview of the Qwen4 Architecture". **MoE: 125B total, 6B activated, plus a 51B n-gram embedding and a 4B MTP head.** 48 layers, hidden size 2560, layout 12 x (3 x Gated DeltaNet + 1 x Qwen Sparse Attention), each layer followed by an MoE block. **512 experts, 10 routed + 1 shared**, expert FFN dim 640. Native context 262144. License `qwen-community-1.0`. The 27B is Apache 2.0 per [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md).
+- **UD-Q4_K_XL = 111.33 GB** in 4 shards (0.01 + 49.86 + 49.38 + 12.09 GB, [HF tree API](https://huggingface.co/api/models/unsloth/Qwen3.8-Flash-Next-GGUF/tree/main?recursive=true)). It splits into:
+  - `per_layer_token_embd` (n-gram table): **28.80 GB** IQ4_NL, read a few rows per token. The same size in every UD quant.
+  - routed experts `ffn_{gate,up,down}_exps`: **77.02 GB**
+  - everything else (attention/DeltaNet, shared experts, routers, hyper-connections, output head): **~4.8 GB**, mostly Q8_0
+- **Bytes per token:** 4.8 + 77.02 x 10/512 (1.50) ≈ **~6.3 GB**, 2.7x less than the 27B Q4. The all-in-VRAM ceiling is 6.3 / 440 → ~70 t/s. Batch-1 MoE kernels gather scattered experts and reach less of peak, so the estimate is **~40-60 t/s if it fit in VRAM**. For scale, Strata reports 63 (prose) / 87 (code) t/s on one R9700 with this file and its own engine (§6).
+- **Quality** (Qwen card, BF16): SWE-bench Pro 62.5 vs 61.7 for the 27B, SWE-bench Multilingual 81.0 vs 73.8, DeepSWE 58.7 vs 42.2, Toolathlon 73.5 vs 67.1. Unsloth's quant table ([run guide](https://unsloth.ai/docs/models/qwen3.8-next)) gives mean KLD: UD-Q2_K_XL 0.225, UD-IQ3_XXS 0.165, UD-Q3_K_XL 0.107, UD-IQ4_XS 0.084, **UD-Q4_K_XL 0.047**.
+
+**Fit in 64 GB VRAM** (non-PLE = total − 28.8 GB, the PLE left on SSD via `--lazy-mode`, §5):
+
+| Quant | Total | Non-PLE (needs VRAM) | Result on 2x32 GB |
+|---|---|---|---|
+| UD-Q4_K_XL | 111.3 | **82.5** | ~20-24 GB of experts must live off-GPU. **Does not fit** |
+| UD-IQ4_XS | 93.7 | 64.9 | ~6-9 GB off-GPU after KV and buffers |
+| UD-Q3_K_XL | 90.0 | 61.2 | Borderline, a few GB off-GPU |
+| UD-IQ3_XXS | 82.0 | 53.2 | **Fits**, ~10 GB headroom |
+| UD-Q2_K_XL | 78.9 | 50.1 | **Fits**, ~13 GB headroom |
+
+KV is small. Only the 12 QSA layers have one: 12 x 2 KV heads x 256 x 2 (K+V) x 2 B ≈ 24.6 KB/token f16, ~6.4 GB at 262144 tokens, ~3.4 GB at q8_0 (estimate; ignores indexer keys).
+
+**llama.cpp support status:**
+
+- Merged: model support [#27742](https://github.com/ggml-org/llama.cpp/pull/27742) (2026-08-27), MTP [#29761](https://github.com/ggml-org/llama.cpp/pull/29761) (2026-10-01), `-sm tensor` re-enabled [#28569](https://github.com/ggml-org/llama.cpp/pull/28569), many QSA fixes through 2026-10-27 (e.g. [#31243](https://github.com/ggml-org/llama.cpp/pull/31243)).
+- **Open bugs that hit this setup:**
+  - [#29149](https://github.com/ggml-org/llama.cpp/issues/29149): **HIP livelock during load on an R9700**, with both UD-IQ3_XXS and UD-Q4_K_XL. The root-cause comment places it in the mmap upload path; "the non-mmap path stages through pinned buffers instead". Try `--load-mode none`.
+  - [#29811](https://github.com/ggml-org/llama.cpp/issues/29811) and [#30860](https://github.com/ggml-org/llama.cpp/issues/30860): MTP asserts at startup.
+  - [#30887](https://github.com/ggml-org/llama.cpp/issues/30887): the DFlash2 drafter crashes on the first prompt (R9700, ROCm).
+  - [#30815](https://github.com/ggml-org/llama.cpp/pull/30815): QSA chunking, still open.
+
+  Usable, but rough on HIP.
+
+## 5. Offloading to DDR RAM and SSD
+
+**What llama.cpp offers** (master `common/arg.cpp`):
+
+- `-ot/--override-tensor <regex>=<buft>` places matching tensors, e.g. `blk\.[0-9]+\.ffn_.*_exps=CPU`. `-cmoe/--cpu-moe` puts all experts on the CPU. `-ncmoe/--n-cpu-moe N` puts the experts of the first N layers on the CPU. These are MoE-only, so they do nothing for the dense 27B (already noted in `docs/architecture.md`).
+- `--moe-cache-mib N`: an LRU cache in VRAM for host-resident experts, merged in [#29887](https://github.com/ggml-org/llama.cpp/pull/29887) (2026-10-07). On Flash-Next Q4_0 the PR measured 4090 25.0 → 39.4 t/s and 5090 30.8 → 67.8 t/s, on an EPYC host with RAM for all experts. "Small batches only (<= 32 tokens)".
+- `-lzm/--lazy-mode {on,auto,off}`: "read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)". `auto` applies to tensors over 4 GiB, which **covers the 28.8 GB PLE table by default** (renamed in [#27969](https://github.com/ggml-org/llama.cpp/pull/27969)). Per [#30772](https://github.com/ggml-org/llama.cpp/pull/30772) (closed unmerged), mapped PLE pages still accumulate in RSS and single-threaded `madvise` can bottleneck NVMe.
+- `-lm/--load-mode {auto,none,mmap,…}`. With mmap, a tensor that fits nowhere is paged from disk implicitly.
+- **Pipeline parallelism turns off as soon as any tensor override is used** (`!has_tensor_overrides()` in `llama-context.cpp`, as described in the open PR [#29963](https://github.com/ggml-org/llama.cpp/pull/29963)). So `-ot`/`-ncmoe` plus two GPUs also loses the layer-split prefill overlap.
+
+**Numbers for this box:**
+
+- **Dense 27B:** no useful offload. Every weight is read every token, and DDR4 is ~51 GB/s against ~440 GB/s VRAM. Each GB moved to RAM costs ~20-25 ms/token on the CPU (estimate). With two cards the Q8_K_XL fits in VRAM anyway. The only RAM tier that helps is `--cache-ram` (prompt cache), as recorded in [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md).
+- **Flash-Next UD-Q4_K_XL, if RAM were free (hypothetical):** ~22 GB of experts on CPU = 29 % of experts, ~0.43 GB read per token from DDR4 at ~40 GB/s ≈ 11 ms. The GPU part ≈ (4.8 + 1.07) / 440 ≈ 13 ms. Layers are sequential, so the parts add up: **~24 ms ≈ ~40 t/s**, more with `--moe-cache-mib` (estimate). Prefill uploads the host experts once per ubatch: 22 GB over PCIe 4.0 x16 at ~25 GB/s ≈ 0.9 s per ubatch, so ~18 s of transfer for a 40k-token prompt at `-ub 2048` (estimate).
+- **Flash-Next UD-Q4_K_XL, on this box:** RAM is full (§1). Even with `--cache-ram` cut to near zero, 32 GB minus OS, OmniRoute (up to 10 GB) and qdrant leaves maybe ~15 GB (estimate), less than the ~22 GB needed. The rest is paged from the CPU-attached Gen4 x4 NVMe. At ~0.43 GB/token and an assumed 3-6 GB/s for ~1-3 MB expert reads, that is **~70-140 ms/token, ~6-12 t/s**. Prefill re-reads ~22 GB per ubatch from disk, so **minutes per 40k-token prompt**. The page cache will also compete with the swap that is already happening. An outside data point: an RX 9060 XT 16 GB with 64 GB DDR4-3200 got "8 t/s on short context. Longer context 3 t/s" on llama.cpp ROCm ([#29149 comment](https://github.com/ggml-org/llama.cpp/issues/29149)). **Not viable.**
+- **PLE table on SSD (any Flash-Next quant):** 32 rows per token (#30772) of 90 B each (Strata `AMD_HIP.md`), each on its own 4 KB page, so ≤128 KB of random IO per token. At 50 t/s that is a few thousand IOPS, trivial. Prefill touches ~32 pages per token, ~1.3 M random reads for 40k tokens. On NVMe at high queue depth that is a few seconds, but single-threaded `madvise` (#30772) can make it much slower. Strata measured +22.5 % prefill with the table in RAM against direct reads (§6). Decode impact: low. Prefill: measure it.
+
+## 6. Strata
+
+**Identity.** [github.com/Niko1221/Strata](https://github.com/Niko1221/Strata): "Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost". C++, **MIT** ([LICENSE](https://github.com/Niko1221/Strata/blob/main/LICENSE)). Its `docs/HOW_IT_WORKS.md` says it is "Built with parts of llama.cpp / ggml (MIT)". It supports Qwen3.8-Flash-Next (ISTA-DASLab GSQ-RCO quants recommended, Unsloth UD-IQ4_XS…Q8_0 supported), Qwen3.6-35B-A3B and DeepSeek V4.1-Flash. **There is no dense Qwen3.8-27B support**, so it cannot replace today's model.
+
+**How it tiers** ([`docs/HOW_IT_WORKS.md`](https://github.com/Niko1221/Strata/blob/main/docs/HOW_IT_WORKS.md), [`docs/MODELS.md`](https://github.com/Niko1221/Strata/blob/main/docs/MODELS.md)):
+
+- **VRAM:** attention/DeltaNet, routers, shared experts, output head, MTP, KV ("from 64K: only its most-read part, the rest streams from RAM"), plus an **expert cache** that fills the rest of VRAM with the most-used experts and adapts at runtime.
+- **RAM:** "all 24,576 experts, pinned". The CPU computes cache misses in place, concurrently with the GPU (AVX-512/AVX2). `--pcie-frac` splits misses between PCIe fetch and CPU compute. On gfx12 it defaults to 0.15 after a measured +9.9 % to +25.6 % decode on an R9700 (`docs/AMD_HIP.md`).
+- **SSD:** the 28.8 GB n-gram table, "read a few rows per token through the OS cache". From 0.2.7 it keeps the table in RAM when there is room for it plus 8 GiB.
+- **Low-RAM mode:** "setup maps them from the model files and keeps in RAM only what the card does not hold … With a small card most experts come from the SSD and it is much slower".
+- **Multi-GPU** ([`docs/MULTI_GPU.md`](https://github.com/Niko1221/Strata/blob/main/docs/MULTI_GPU.md)): pipeline (layer) split, "not tensor parallelism". "No NVLink or peer-to-peer access is needed, cards on x4 or x1 slots work". Each card caches the experts of its own layers, so two cards hold about twice as many experts. `docs/AMD_HIP.md` documents HIP layer split for gfx1201 pairs and adds "A split is worth it when no single card holds the model's experts".
+
+**AMD/ROCm:** the R9700 (gfx1201) is "Maintainer-tested", with Linux `hip-rx9070` engine packages in every recent release ([releases](https://github.com/Niko1221/Strata/releases)).
+
+**RAM requirement for the requested file:** `docs/MODELS.md` lists UD-Q4_K_XL as "111 GB download, 72 GiB experts, **RAM for all experts ~88 GB**". With less RAM it "keeps as many as fit and reads the rest from the SSD (much slower)".
+
+**Claimed vs verifiable:**
+
+| Claim | Source | Host RAM | Verifiable? |
+|---|---|---|---|
+| UD-Q4_K_XL on one R9700: 63 t/s prose, 87 code, 32K prompt 1,110-1,320 t/s | `docs/MODELS.md`, "engines before 0.1.45.3" | not stated there | Self-reported |
+| IQ3_S on one R9700: 87.6-101.9 t/s across 1K-128K | [community R9700 report](https://github.com/Niko1221/Strata/blob/main/bench/results/2026-10-08-community-r9700-linux/README.md) | **~503 GiB**, 2x EPYC 9334 | Has per-request JSON and hashes. "Single GPU still uses CPU experts, system RAM and PCIe; these rates do not establish desktop-CPU speed or minimum RAM" |
+| R9700 + 9070 XT, 47 GB RAM, Coder IQ1_M: 45.5 / 48.3 t/s decode | `docs/AMD_HIP.md` | 47 GB | Self-reported |
+| "125B+ MoE on a single 12GB GPU", "25-38 tok/s" | [stratallm.org](https://stratallm.org/) | n/a | The site calls itself "Unofficial Community Documentation". Its speed figures contradict each other and link no method. Ignore it |
+
+None of these numbers were rerun for this doc. The project says its benchmark protocol "is still a draft" (`docs/COMMUNITY_BENCHMARKS.md`).
+
+**Maturity:** repo created 2026-09-24. ~4,690 commits, ~3,594 of them by the one maintainer. ~36k stars, 895 open issues, a release roughly daily (0.1.40 → 0.2.7 in three weeks). Installation is `curl … | sh`, an AppImage, or Docker (`docs/DOCKER.md`, not read). It is young, single-maintainer and churning, and it would need its own container and gateway wiring here.
+
+**Verdict for this box:** not worth it now. The 32 GB of RAM defeats its core design (all experts pinned in RAM). Low-RAM mode on 2x32 GB VRAM would still read ~20 GB of experts from NVMe for UD-Q4_K_XL. It cannot run the dense 27B that this stack is tuned around. **Revisit if RAM goes to 96-128 GB** (the board takes 128 GB). On a big-RAM host its self-reported R9700 Flash-Next numbers (60-100 t/s) are well above llama.cpp's 27B, so it would be worth a real A/B then.
+
+## 7. Recommendation
+
+**Primary: keep `Qwen3.8-27B-UD-Q4_K_XL` + DFlash2 and run two single-GPU replicas, one per card.**
+
+Why:
+
+- (1) Layer split gives a dense model no decode gain.
+- (2) Tensor split is experimental on ROCm, measured at 1.31x, crashes with DFlash2, and would run over a PCIe 3.0 x4 chipset link.
+- (3) Q8_K_XL roughly halves decode.
+- (4) Flash-Next at a usable quant does not fit, and RAM offload pages from NVMe.
+- (5) The workload is 2 parallel agents. The log already shows decode falling to p10 15 t/s while the other slot prefills ([`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md)). Two replicas remove that interference and double each slot's context.
+
+Per-instance flags (two services, identical except the device). Each is today's command with these changes:
+
+```text
+--device ROCm0               # ROCm1 for the second service; confirm names with --list-devices
+--n-gpu-layers 999
+--ctx-size 262144 --parallel 1   # full 262144 per agent instead of 131072
+--flash-attn on --cache-type-k q8_0 --cache-type-v q8_0
+--cache-ram 6144             # per instance; both together must stay well under free RAM (§1)
+--spec-type draft-dflash -md /models/Qwen3.8-27B-DFlash2-Q4_K_M.gguf --spec-draft-n-max 7
+--jinja
+```
+
+`GPU_MAX_HW_QUEUES=1` stays on both services (`docs/architecture.md#gpu-access`). Gateway: register both backends in OmniRoute and make sure a conversation stays on one backend (prompt-cache affinity). Otherwise every switch is a full re-prefill. Whether OmniRoute can balance two connections per conversation was not checked.
+
+**Independently of the second card: lower `LLAMA_CACHE_RAM`.** 24 GiB on a 32 GB host with a 10 GB OmniRoute limit oversubscribes RAM.
+
+**Alternatives to A/B, in order:**
+
+1. **Single fast stream:** Q4_K_XL with `-sm tensor -ts 1,1 -fa on --fit off --ctx-size <set by hand> --spec-type draft-mtp --spec-draft-n-max 2` (the built-in MTP head; DFlash2 crashes with `tensor`). Estimate ~34 t/s no spec, ~50 with MTP if stable. Watch for #27122-style lockups.
+2. **Quality:** Q8_K_XL (or UD-Q6_K_XL at 25.3 GB) with `-sm layer`, DFlash2 placed with `-devd`, ~25 t/s estimated. Only if Q4 quality is shown to fail on your tasks.
+3. **Flash-Next experiment:** UD-IQ3_XXS or UD-Q2_K_XL, all in VRAM: `-sm layer -ngl 999 --lazy-mode auto --load-mode none -fa on -c 131072`, MTP via `-md MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf --spec-type draft-mtp` once #29811/#30860 are fixed. Judge it on your own agent tasks. The KLD at these sizes is 3.5-5x that of UD-Q4_K_XL.
+
+## 8. What to benchmark once the card arrives
+
+1. **Topology:** `sudo lspci -vv` LnkSta for both cards (expect 16 GT/s x16 and 8 GT/s x4). Run `rocm-smi --showtopo` and `rocm-bandwidth-test` for host↔GPU and GPU↔GPU (does P2P work through the chipset?). Check the PSU and the case slot fit first.
+2. **Per-card baseline:** `llama-bench -m Qwen3.8-27B-UD-Q4_K_XL.gguf -dev ROCm0` and then `-dev ROCm1`, `-fa 1 -ctk q8_0 -ctv q8_0 -p 512,8192 -n 128 -d 0,32768`. Decode should match. Load time and prompt-cache restores are where the x4 link shows.
+3. **Split modes:** the same bench with `-sm layer` vs `-sm tensor -ts 1,1` for Q4_K_XL and Q8_K_XL. Compare tg against #28863's 1.31x and pp against layer.
+4. **Server replay** (fixed ~40k and ~2k prompts, production temperature, concurrency 1 and 2) for: (a) two replicas, (b) one `-sm layer --parallel 2`, (c) `-sm tensor` + MTP. Record `timings.predicted_per_second`, `prompt_per_second`, draft acceptance, and the p10 decode under concurrency.
+5. **Correctness:** a needle test at ~6-10k and ~60-90k tokens on each config, because #30884 reproduces on 2x R9700 with `--tensor-split`. Also check that DFlash2 output under `-sm layer` matches single-card output on a fixed prompt.
+6. **RAM:** `free -h` and `vmstat 5` after an hour of agent traffic, before and after lowering `--cache-ram`. Swap-in should be ~0.
+7. **Optional Flash-Next:** that UD-IQ3_XXS/UD-Q2_K_XL loads at all on HIP (#29149, try `--load-mode none`), tg/pp at 4k/32k, and prefill time with the PLE table on NVMe.
+
+## Open questions
+
+- Does OmniRoute support two backends with sticky routing per conversation?
+- Does the `ghcr.io/ggml-org/llama.cpp:server-rocm` image build with `GGML_HIP_RCCL`? It only matters for `-sm tensor`.
+- Does ROCm P2P work between a CPU-attached slot and a chipset-attached slot on B550? It is unverified, and the answer probably does not matter outside `tensor` mode.
+- Is there a B550/X570 board with CPU x8/x8 bifurcation that fits this chassis? It would give both cards Gen4 x8. It is only worth it if step 3 shows `tensor` scaling well. No specific board was checked.
+
+## Sources
+
+- Hugging Face: [unsloth/Qwen3.8-27B-GGUF tree](https://huggingface.co/api/models/unsloth/Qwen3.8-27B-GGUF/tree/main), [its config.json](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/raw/main/config.json), [UD-Q8_K_XL file](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF?show_file_info=Qwen3.8-27B-UD-Q8_K_XL.gguf), [unsloth/Qwen3.8-Flash-Next-GGUF](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF) and its [tree](https://huggingface.co/api/models/unsloth/Qwen3.8-Flash-Next-GGUF/tree/main?recursive=true), [UD-Q4_K_XL shard 1](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF?show_file_info=UD-Q4_K_XL%2FQwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf), [z-lab DFlash2 GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF), [incoai Flash-Next DFlash2 GGUF](https://huggingface.co/incoai/Qwen3.8-Flash-Next-DFlash2-GGUF). Tensor sizes come from parsing the GGUF headers (range requests) with a throwaway script.
+- Unsloth: [Qwen3.8-Flash-Next run guide](https://unsloth.ai/docs/models/qwen3.8-next) (sizes, KLD table, MTP flags).
+- llama.cpp master (fetched 2026-10-29): [`docs/multi-gpu.md`](https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md), [`common/arg.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/common/arg.cpp), [`src/llama-arch.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/src/llama-arch.cpp), [`src/llama-context.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/src/llama-context.cpp). PRs: [#19378](https://github.com/ggml-org/llama.cpp/pull/19378), [#23792](https://github.com/ggml-org/llama.cpp/pull/23792), [#27342](https://github.com/ggml-org/llama.cpp/pull/27342), [#27742](https://github.com/ggml-org/llama.cpp/pull/27742), [#27825](https://github.com/ggml-org/llama.cpp/pull/27825), [#27858](https://github.com/ggml-org/llama.cpp/pull/27858), [#27861](https://github.com/ggml-org/llama.cpp/pull/27861), [#27969](https://github.com/ggml-org/llama.cpp/pull/27969), [#28569](https://github.com/ggml-org/llama.cpp/pull/28569), [#29761](https://github.com/ggml-org/llama.cpp/pull/29761), [#29887](https://github.com/ggml-org/llama.cpp/pull/29887), [#29963](https://github.com/ggml-org/llama.cpp/pull/29963), [#30772](https://github.com/ggml-org/llama.cpp/pull/30772), [#30815](https://github.com/ggml-org/llama.cpp/pull/30815), [#31243](https://github.com/ggml-org/llama.cpp/pull/31243). Issues: [#23567](https://github.com/ggml-org/llama.cpp/issues/23567), [#27122](https://github.com/ggml-org/llama.cpp/issues/27122), [#27829](https://github.com/ggml-org/llama.cpp/issues/27829), [#28582](https://github.com/ggml-org/llama.cpp/issues/28582), [#28777](https://github.com/ggml-org/llama.cpp/issues/28777), [#28863](https://github.com/ggml-org/llama.cpp/issues/28863), [#29149](https://github.com/ggml-org/llama.cpp/issues/29149), [#29811](https://github.com/ggml-org/llama.cpp/issues/29811), [#30860](https://github.com/ggml-org/llama.cpp/issues/30860), [#30884](https://github.com/ggml-org/llama.cpp/issues/30884), [#30887](https://github.com/ggml-org/llama.cpp/issues/30887).
+- Hardware: [ASRock B550M Pro4 manual](https://download.asrock.com/Manual/B550M%20Pro4.pdf) (asrock.com's spec page returned no content to the fetcher), [ComputerBase B550](https://www.computerbase.de/2020-06/amd-b550-chipsatz-mainboard-test/), [Guru3D B550 PCIe](https://www.guru3d.com/review/amd-ryzen-3-3100-and-3300x-review/page-3/), [Puget R9700](https://www.pugetsystems.com/parts/Video-Card/AMD-Radeon-AI-PRO-R9700-32GB-16908/), [Tom's Hardware R9700](https://www.tomshardware.com/pc-components/gpus/amd-launches-radeon-ai-pro-r9700-to-challenge-nvidias-ai-market-dominance).
+- Strata: [repo](https://github.com/Niko1221/Strata) (README, LICENSE, `docs/HOW_IT_WORKS.md`, `MODELS.md`, `MULTI_GPU.md`, `SECOND_GPU.md`, `AMD_HIP.md`, `COMMUNITY_BENCHMARKS.md`, read via the GitHub API on 2026-10-29), [releases](https://github.com/Niko1221/Strata/releases), [stratallm.org](https://stratallm.org/).
+- This repo: `docker-compose.yml`, `.env.example`, `docs/architecture.md`, [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md), [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md), [`500k-context-2-parallel-agents-gpu-roadmap.md`](500k-context-2-parallel-agents-gpu-roadmap.md).

docs/research/image-generation-model-choice.md

@@ -1,220 +0,0 @@
-# Research: which diffusion model to target with the full ~32GB R9700
-
-**Question:** With Qwen/llama-server fully stopped (per issue #39's premise — see
-map #38), ComfyUI has the whole ~32GB R9700 (ROCm/gfx1201) to itself instead
-of the ~6GB left over during concurrent operation
-(per [`image-generation-options.md`](image-generation-options.md)). Given that
-headroom, should the build stay on FLUX.1-schnell, or move up to
-FLUX.1-dev, SD3.5-large, Qwen-Image, HunyuanImage-3.0, or Krea-2?
-
-**Answer, short version:** move up to **Qwen-Image at FP8 precision**
-(`qwen_image_fp8_e4m3fn.safetensors` diffusion weights +
-`qwen_2.5_vl_7b_fp8_scaled.safetensors` text encoder, ~25GB combined). It's
-the only one of the five upgrade candidates with **direct, hardware-specific
-evidence of running on this exact GPU architecture** (gfx1201/R9700) rather
-than a generic "ComfyUI supports ROCm" inference, it carries the cleanest
-license of the group (Apache-2.0, no revenue threshold, no non-commercial
-clause), and its 20B MMDiT is a real capability step up from schnell's
-distilled 12B (notably for text rendering and prompt adherence), while still
-fitting with real margin inside 32GB.
-
-## Why not just re-derive the schnell/dev/SDXL/SD3.5 findings
-
-`docs/research/image-generation-options.md` already covers, with primary
-sources: ComfyUI's ROCm/gfx1201 story (official AMD docs + RDNA4 blog post +
-community gfx1201 Docker images), FLUX.1-schnell vs FLUX.1-dev vs SDXL vs
-SD3.5 licenses, and FLUX GGUF VRAM figures at the ~6GB-headroom scale. None
-of that is repeated here except where the ~32GB ceiling changes the
-conclusion. This doc adds: FLUX.1-dev/SD3.5 at the *larger* headroom, plus
-three models the prior doc didn't cover at all (Qwen-Image, HunyuanImage-3.0,
-Krea-2).
-
-## Candidate comparison
-
-| Model | License (primary source) | Params | Stated/typical VRAM | ROCm/gfx1201 evidence |
-|---|---|---|---|---|
-| FLUX.1-schnell (current) | Apache-2.0 | 12B | GGUF Q4_K_S ~7GB (per prior doc) | Confirmed on gfx1201 (prior doc) |
-| **Qwen-Image** | **Apache-2.0** | 20B (20.4B DiT + 8.3B Qwen2.5-VL text encoder) | fp8 ~16GB (diffusion) + ~9.4GB (fp8 text encoder) ≈ 25GB total; bf16 needs 24GB+ and "48GB+" per some quant write-ups | **Direct**: [kyuz0/amd-r9700-comfy](https://github.com/kyuz0/amd-r9700-comfy) ships a pre-validated "Qwen Image 2512 (FP8) & Lightning LoRA (4 steps)" ComfyUI workflow specifically for the R9700 AI Pro (gfx1201), on a ROCm 7 (TheRock nightlies) toolbox |
-| FLUX.1-dev | [FLUX.1-dev Non-Commercial License](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md) — non-commercial weights, outputs usable commercially | 12B | bf16 ~24GB; GGUF Q8 ~12-13GB | Same *family* evidence as schnell (gfx1201 Docker images target FLUX generally), but no R9700-specific FLUX.1-dev report found |
-| SD3.5-large | [Stability Community License](https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/LICENSE.md) — free under $1M annual revenue | 8B | bf16 ~16GB; 4-bit NF4 fits small GPUs | AMD's own ComfyUI-ROCm doc lists an "SD3.5 Simple" template workflow (per prior doc) — vendor-blessed but not R9700-specific |
-| HunyuanImage-3.0 | [tencent-hunyuan-community license](https://github.com/Tencent-Hunyuan/HunyuanImage-3.0) | 80B total / 13B active (MoE, 64 experts) | Official repo: "≥ 3 × 80GB" VRAM for the base model, "≥ 8 × 80GB" for -Instruct; ~177GB at fp16 | **None, and actively contraindicated**: setup requires CUDA 12.8 + FlashAttention2/FlashInfer, no AMD/ROCm mention anywhere in the official repo |
-| Krea-2 (Turbo) | [Krea 2 Community License](https://www.krea.ai/krea-2-licensing) — free under $1M annual revenue, homelab/personal explicitly covered | 12-13B DiT | No official VRAM figure; community reports (RTX hardware only) cite fp8 ~16GB, GGUF ~12GB | **None found** — released [June 22, 2026 per the HF model card](https://huggingface.co/krea/Krea-2-Turbo); ComfyUI added native support per [blog.comfy.org](https://blog.comfy.org/p/krea-2-open-source-models-are-now), and GGUF quants exist ([molbal/krea2-gguf](https://huggingface.co/molbal/krea2-gguf)), but no R9700/gfx1201-specific report exists yet — too new for that evidence to have accumulated |
-
-## Per-model detail
-
-### Qwen-Image — recommended
-
-- **License**: Apache-2.0, stated directly on the model card, no revenue
-  threshold, no non-commercial clause, no attribution/naming requirement.
-  The cleanest license of every model considered in this doc or the prior
-  one. Source: [Qwen/Qwen-Image on Hugging Face](https://huggingface.co/Qwen/Qwen-Image).
-- **Architecture**: 20B-parameter MMDiT (Multimodal Diffusion Transformer)
-  combined with an 8.3B Qwen2.5-VL text encoder — notably larger and more
-  capable than FLUX.1-schnell's 12B distilled model, particularly for
-  multilingual text rendering and instruction-following, per the official
-  [QwenLM/Qwen-Image GitHub repo](https://github.com/QwenLM/Qwen-Image).
-- **ComfyUI support**: native, not a wrapper/custom-node integration —
-  landed August 2025 per [ComfyUI Wiki's native-support announcement](https://comfyui-wiki.com/en/news/2025-08-05-qwen-image),
-  with an official FP8 checkpoint (`qwen_image_fp8_e4m3fn.safetensors`) and
-  FP8-scaled text encoder (`qwen_2.5_vl_7b_fp8_scaled.safetensors`, 9.38GB)
-  published under [Comfy-Org/Qwen-Image_ComfyUI](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders).
-- **VRAM at FP8**: the official Comfy-Org FP8 diffusion checkpoint plus FP8
-  text encoder land around ~25GB combined — comfortably inside the 32GB
-  ceiling with headroom for ComfyUI runtime/VAE overhead, versus 24GB+ (some
-  sources say 48GB+) for the unquantized bf16 model. Sources: community
-  VRAM write-ups aggregated via [Comfy-Org/ComfyUI issue #10852 ("Qwen-image in 24GB VRAM and 32GB RAM")](https://github.com/Comfy-Org/ComfyUI/issues/10852)
-  and the [official ComfyUI Qwen-Image example](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/) —
-  treat exact GB figures as secondary/community-sourced, consistent with how
-  the prior doc flagged FLUX's GGUF VRAM table.
-- **Direct R9700/gfx1201 evidence (the deciding factor)**:
-  [kyuz0/amd-r9700-comfy](https://github.com/kyuz0/amd-r9700-comfy) is a
-  Fedora-toolbox ROCm 7 (TheRock nightlies) environment built specifically
-  for the "AMD Radeon 9700 AI PRO (32GB)" and ships a pre-validated
-  "Qwen Image 2512 (FP8) & Lightning LoRA (4 steps)" ComfyUI workflow
-  (plus a Qwen-Image-Edit 2511 FP8 workflow) as one of only four workflows
-  in the whole repo. This is the only model in this comparison with
-  card-architecture-specific (not just "ComfyUI supports ROCm generically")
-  validation — everything else relies on family-level or vendor-generic
-  ROCm claims. A related community discussion
-  ([pollockjj/ComfyUI-MultiGPU #133](https://github.com/pollockjj/ComfyUI-MultiGPU/discussions/133))
-  does flag that "Qwen image edit doesn't always work on AMD HIP/ROCm" in
-  some configurations — worth testing the specific workflow before assuming
-  zero friction, but this is a known-quantity, actively-discussed rough edge
-  rather than a documented hard blocker.
-- **4-step Lightning LoRA**: the R9700-validated workflow pairs Qwen-Image
-  with a "Lightning" LoRA for 4-step inference — the same fast-inference
-  pattern FLUX.1-schnell uses, so switching models doesn't have to mean
-  giving up the short GPU-resident-time-per-image property the prior doc
-  called out as valuable for time-sliced use alongside llama-server (even
-  though this ticket's premise is llama-server being stopped, that pattern
-  still helps if the two services are ever run in an overlapping window).
-
-### FLUX.1-dev — solid alternative, not the pick
-
-- License permits non-commercial use of the weights; BFL's own license page
-  states generated outputs are separately usable commercially (already
-  covered in the prior doc; unchanged here). Source:
-  [FLUX.1-dev LICENSE.md](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md).
-- At bf16 (~24GB) or GGUF Q8 (~12-13GB), it fits the 32GB ceiling with room
-  to spare — a legitimate move up from schnell's distilled quality.
-  Source: [city96/FLUX.1-dev-gguf](https://huggingface.co/city96/FLUX.1-dev-gguf)
-  and community Q8 VRAM reports.
-- Loses to Qwen-Image on two counts: license (non-commercial-weights clause
-  vs. Apache-2.0 — not a hard blocker for this homelab per the prior doc's
-  own reasoning, but strictly worse) and hardware evidence (FLUX's gfx1201
-  validation is at the *family* level — the prior doc's `yurisasc/comfyui-rocm-rdna4`
-  and `charlie12345/R9700AIProComfyUIPatch` sources are about running FLUX
-  models on this card generally, not a FLUX.1-dev-specific report the way
-  kyuz0's repo is Qwen-Image-specific).
-
-### SD3.5-large — no longer the best use of the freed headroom
-
-- License unchanged from the prior doc: Stability Community License, free
-  under $1M annual revenue (irrelevant threshold for this homelab). Source:
-  [stabilityai/stable-diffusion-3.5-large LICENSE.md](https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/LICENSE.md).
-- At 8B params (bf16 ~16GB), it's the smallest of the upgrade candidates —
-  which mattered when 6GB was the ceiling, but with 32GB available there's
-  no VRAM reason to pick the model the prior doc already flagged as "lower
-  fidelity than FLUX/SD3.5 by current standards" over Qwen-Image or
-  FLUX.1-dev. AMD's own ComfyUI-ROCm docs do list it as a first-party
-  example template ("SD3.5 Simple", per the prior doc), so it remains a fine
-  fallback if Qwen-Image's FP8 path hits the ROCm rough edge noted above.
-
-### HunyuanImage-3.0 — ruled out
-
-- Official repo states VRAM requirements of "≥ 3 × 80GB" for the base model
-  and "≥ 8 × 80GB" for HunyuanImage-3.0-Instruct — i.e. multi-GPU
-  datacenter-class NVIDIA clusters, not a single 32GB consumer/workstation
-  card at any precision. Source:
-  [Tencent-Hunyuan/HunyuanImage-3.0 GitHub repo](https://github.com/Tencent-Hunyuan/HunyuanImage-3.0).
-- Setup instructions require CUDA 12.8, PyTorch 2.8.0 built for CUDA, and
-  optionally FlashAttention2/FlashInfer for MoE routing speed — no AMD or
-  ROCm path is mentioned anywhere in the official repo. Even the
-  MoE-efficient "13B active" framing doesn't help here: the tooling itself
-  assumes an NVIDIA multi-GPU cluster, and the 80B total parameter set still
-  has to be resident somewhere.
-- 32GB of headroom on one AMD card doesn't move this model into reach at
-  any precision considered here; it's excluded regardless of how much VRAM
-  frees up on this specific box.
-
-### Krea-2 (Turbo) — promising, but unverified on this hardware
-
-- Verified directly against primary sources per the ticket's instruction
-  (this is a June 2026 release, past most training cutoffs): the
-  [Hugging Face model card](https://huggingface.co/krea/Krea-2-Turbo) states
-  a release date of **June 22, 2026**, a 12-billion-parameter single-stream
-  diffusion transformer, `torch.bfloat16` as the reference precision, and
-  the **Krea 2 Community License**.
-- License, per [krea.ai/krea-2-licensing](https://www.krea.ai/krea-2-licensing):
-  non-commercial (including explicitly personal/homelab) use is free;
-  commercial use is permitted royalty-free for entities under $1M
-  trailing-12-month revenue (same shape as SD3.5's and matching this
-  homelab's use case); content-filter and AI-disclosure obligations apply if
-  deployed publicly; derivative model names must start with "Krea".
-- ComfyUI added native support for both open-weight checkpoints (Krea 2 Raw
-  and Krea 2 Turbo) per [blog.comfy.org's announcement](https://blog.comfy.org/p/krea-2-open-source-models-are-now),
-  and community GGUF quants already exist
-  ([molbal/krea2-gguf](https://huggingface.co/molbal/krea2-gguf)), with
-  reports (RTX hardware only) of fp8 fitting 16GB and GGUF fitting 12GB.
-- **No AMD/ROCm mention anywhere** in the model card, and no gfx1201/R9700
-  community report was found — unsurprising given the model is roughly
-  2.5 months old at the time of this research. Its architecture (a
-  standard-shaped DiT that ComfyUI loads through its normal diffusion-model
-  nodes, per the ComfyUI blog post) gives reasonable expectation it will run
-  on the same ROCm/PyTorch backend already proven for FLUX and Qwen-Image on
-  this card, but that's an inference, not a verified fact the way
-  kyuz0's Qwen-Image workflow is.
-- **Not the pick today**, precisely because Qwen-Image already offers a
-  hardware-verified path at a comparable parameter count and VRAM budget.
-  Worth a follow-up research ticket once R9700/gfx1201-specific Krea-2
-  reports exist — the license and ComfyUI support are both already in
-  place, so the only open question is real-world ROCm behavior.
-
-## Recommendation
-
-**Qwen-Image, FP8 precision** (`qwen_image_fp8_e4m3fn.safetensors` +
-`qwen_2.5_vl_7b_fp8_scaled.safetensors`, ~25GB combined), optionally paired
-with the 4-step Lightning LoRA the R9700-specific validated workflow uses.
-It wins on all three axes the ticket asked about:
-
-1. **License**: Apache-2.0 — no restriction at all, strictly better than
-   every other candidate including the current FLUX.1-schnell pick.
-2. **ROCm/gfx1201 compatibility**: the only candidate with a workflow
-   pre-validated specifically on this GPU architecture
-   ([kyuz0/amd-r9700-comfy](https://github.com/kyuz0/amd-r9700-comfy)),
-   not just "ComfyUI supports ROCm in general."
-3. **VRAM at the ~32GB ceiling**: ~25GB at FP8 leaves real margin for
-   ComfyUI runtime/VAE overhead, without needing the multi-step,
-   non-distilled FLUX.1-dev's full 24GB bf16 footprint or accepting
-   SD3.5's lower fidelity ceiling — and it's a genuine capability upgrade
-   over schnell (20B vs. 12B, non-distilled-quality text rendering) rather
-   than just a bigger file.
-
-If the known Qwen-Image/ROCm edit-mode rough edge
-([pollockjj/ComfyUI-MultiGPU #133](https://github.com/pollockjj/ComfyUI-MultiGPU/discussions/133))
-turns out to affect plain text-to-image generation too, SD3.5-large (AMD's
-own first-party "SD3.5 Simple" ComfyUI-ROCm template workflow) is the
-fallback, with FLUX.1-dev as a second option. HunyuanImage-3.0 is excluded
-outright regardless of available VRAM (CUDA-only tooling, multi-GPU
-datacenter VRAM floor). Krea-2 is worth revisiting once R9700-specific
-field reports exist.
-
-## Sources consulted
-
-- [docs/research/image-generation-options.md](image-generation-options.md) (this repo — prior findings, not re-derived)
-- [Qwen/Qwen-Image (Hugging Face)](https://huggingface.co/Qwen/Qwen-Image)
-- [QwenLM/Qwen-Image (GitHub)](https://github.com/QwenLM/Qwen-Image)
-- [ComfyUI Wiki — Qwen-Image native support announcement](https://comfyui-wiki.com/en/news/2025-08-05-qwen-image)
-- [Comfy-Org/Qwen-Image_ComfyUI (Hugging Face, FP8 checkpoints)](https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/tree/main/split_files/text_encoders)
-- [Comfy-Org/ComfyUI issue #10852 — Qwen-Image VRAM](https://github.com/Comfy-Org/ComfyUI/issues/10852)
-- [ComfyUI official Qwen-Image example](https://comfyanonymous.github.io/ComfyUI_examples/qwen_image/)
-- [kyuz0/amd-r9700-comfy (R9700-specific ROCm ComfyUI toolbox)](https://github.com/kyuz0/amd-r9700-comfy)
-- [pollockjj/ComfyUI-MultiGPU discussion #133 (Qwen-Image-Edit ROCm rough edge)](https://github.com/pollockjj/ComfyUI-MultiGPU/discussions/133)
-- [black-forest-labs/FLUX.1-dev (Hugging Face) + LICENSE.md](https://huggingface.co/black-forest-labs/FLUX.1-dev)
-- [city96/FLUX.1-dev-gguf](https://huggingface.co/city96/FLUX.1-dev-gguf)
-- [stabilityai/stable-diffusion-3.5-large (Hugging Face) + LICENSE.md](https://huggingface.co/stabilityai/stable-diffusion-3.5-large)
-- [Tencent-Hunyuan/HunyuanImage-3.0 (GitHub)](https://github.com/Tencent-Hunyuan/HunyuanImage-3.0)
-- [krea/Krea-2-Turbo (Hugging Face)](https://huggingface.co/krea/Krea-2-Turbo)
-- [Krea 2 Community License Agreement (krea.ai)](https://www.krea.ai/krea-2-licensing)
-- [blog.comfy.org — Krea 2 open-source models in ComfyUI](https://blog.comfy.org/p/krea-2-open-source-models-are-now)
-- [molbal/krea2-gguf (Hugging Face)](https://huggingface.co/molbal/krea2-gguf)

docs/research/image-generation-options.md

@@ -1,249 +0,0 @@
-# Research: adding local image generation to the stack
-
-**Question:** What's the best way to add local image generation alongside
-the existing Qwen3.8-27B / llama.cpp text stack, given a single AMD Radeon
-R9700 (32GB VRAM, ROCm/gfx1201 — not CUDA), routed through the OmniRoute
-gateway on the `ai-stack` Docker network?
-
-**Answer, short version:** run **ComfyUI** (official AMD-blessed ROCm
-Docker path exists, and OmniRoute already has a first-class `comfyui`
-provider — no bespoke API wrapper needed) with **FLUX.1 [schnell]**
-(Apache-2.0, 4-step, GGUF-quantizable) as the default model, falling back to
-**SDXL** for anything schnell's distilled-step license/quality tradeoffs
-don't suit. VRAM headroom against the current llama-server footprint is too
-tight for both to be resident at once at any real image quality — plan for
-**time-sliced use** (llama-server's existing lazytainer stop-on-idle pattern,
-mirrored for the image-gen service, or a manual "stop one, start the other"
-toggle), not concurrent operation.
-
-## Current VRAM baseline (from this repo)
-
-Per `docker-compose.yml` and `.env.example`, llama-server runs
-`Qwen3.8-27B-UD-Q4_K_XL.gguf` (17.6 GB weights) at `--ctx-size 262144` with
-`--cache-type-k q8_0 --cache-type-v q8_0`, landing at **~25.6 GB** total
-(weights + q8_0 KV cache), leaving **~6 GB** free on the 32GB card — this
-matches the math already recorded in
-[`docs/research/qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md). Per the auto-memory
-note on this repo, real measured VRAM use has run closer to ~75% (~24 GB) in
-practice versus the theoretical estimate, which doesn't change the
-conclusion below but means the ~6 GB figure is closer to a ceiling than a
-comfortable number.
-
-**Implication:** 6 GB is not enough for any current-generation image model at
-usable quality (see VRAM table below — even the smallest practical FLUX
-quant wants ~7 GB alone, before ComfyUI's own runtime/VAE overhead). Running
-image-gen *concurrently* with llama-server resident is not realistic on this
-card. The two need to time-share the GPU, not split it.
-
-## Backend evaluation (ROCm support, checked against primary sources)
-
-### ComfyUI — recommended
-
-- **Official AMD ROCm docs exist and are current.** AMD's own ROCm docs site
-  hosts a dedicated ComfyUI install guide with a prebuilt Docker image path
-  (recommended) or build-from-source, listing ROCm 7.2.0 and 7.1.0 as
-  supported versions, explicit `--device=/dev/kfd --device=/dev/dri
-  --group-add video` flags (same device-passthrough pattern this repo
-  already uses for llama-server), and template workflows including "SD3.5
-  Simple". Officially the guide only names AMD Instinct
-  MI355X/MI325X/MI300X (datacenter cards) as supported platforms.
-  Source: [ROCm docs — ComfyUI on ROCm installation](https://rocm.docs.amd.com/projects/comfyui/en/docs-26.04/install/comfyui-install.html).
-- **The upstream ComfyUI README itself documents AMD support directly**,
-  including consumer cards: stable ROCm install via
-  `pip install torch torchvision torchaudio --index-url
-  https://download.pytorch.org/whl/rocm7.2`, plus an experimental Windows
-  build explicitly naming **RDNA 3 (RX 7000), RDNA 3.5 (Strix Halo), and
-  RDNA 4 (RX 9000 series)** — i.e. the same RDNA4 generation as the R9700 —
-  and `HSA_OVERRIDE_GFX_VERSION` workarounds for older/unlisted cards.
-  Source: [comfyanonymous/ComfyUI README](https://github.com/comfyanonymous/ComfyUI).
-- **AMD has published a specific RDNA4/RX 9000 ComfyUI guide** (separate
-  from the Instinct-only install page above), confirming RDNA4 consumer
-  cards are an explicitly supported, first-party-documented target, not just
-  a community workaround.
-  Source: [ROCm blog — Getting Started with ComfyUI on AMD Radeon RX 9000 Series GPUs](https://rocm.blogs.amd.com/artificial-intelligence/comfyui-radeon-9000/README.html).
-- **gfx1201 (R9700's arch) specifically has active community Docker images**:
-  `yurisasc/comfyui-rocm-rdna4` targets ROCm 7.1 + PyTorch 2.9.1 with
-  `HSA_OVERRIDE_GFX_VERSION=12.0.1` / `PYTORCH_ROCM_ARCH=gfx1201` baked in,
-  and there's a published community patch specifically for R9700 AI Pro +
-  ComfyUI video-gen speedups, evidence the card is being run today, not just
-  theoretically compatible.
-  Sources: [yurisasc/comfyui-rocm-rdna4](https://github.com/yurisasc/comfyui-rocm-rdna4),
-  [charlie12345/R9700AIProComfyUIPatch](https://github.com/charlie12345/R9700AIProComfyUIPatch).
-- **Known gfx1201 caveat:** AMD's own TransformerEngine repo has an open
-  issue confirming gfx1201 is missing from the FP8 architecture table, so
-  FP8 kernels silently fall back to FP32 with ~50% throughput loss
-  (18-22 vs. 35-40 tok/s in the reporter's LLM benchmark) — not a
-  correctness blocker, but relevant if planning to use FP8-quantized image
-  models expecting native FP8 speed on this card; GGUF/Q-quants (see below)
-  avoid this path entirely since they dequantize to bf16/fp16, not fp8.
-  Source: [ROCm/TransformerEngine issue #520](https://github.com/ROCm/TransformerEngine/issues/520).
-- **Actively maintained community Docker packaging** beyond AMD's own image:
-  `YanWenKun/ComfyUI-Docker` ships parallel `rocm` (PyTorch-build-based,
-  faster releases) and `rocm7` (AMD-build-based, more comprehensive)
-  variants, both targeting ROCm 7, with ~1000 commits of ongoing history —
-  a viable alternative to the official AMD image if it lags behind ComfyUI
-  releases.
-  Source: [YanWenKun/ComfyUI-Docker](https://github.com/YanWenKun/ComfyUI-Docker).
-
-### AUTOMATIC1111 / Forge — usable but a step down for this hardware
-
-- ROCm support for A1111/Forge is real but community-patched, not
-  first-party. The upstream `lllyasviel/stable-diffusion-webui-forge` repo's
-  own discussion thread on AMD support points users to
-  `lshqqytiger/stable-diffusion-webui-amdgpu-forge`, a community fork
-  specifically maintained for AMD, "regarded as the go-to version" for
-  running FLUX-era models on AMD — i.e. the *mainline* Forge repo does not
-  claim ROCm support itself; you're expected to run a fork.
-  Source: [lllyasviel/stable-diffusion-webui-forge discussion #67](https://github.com/lllyasviel/stable-diffusion-webui-forge/discussions/67).
-- No first-party AMD vendor documentation (unlike ComfyUI's AMD-authored
-  ROCm/RDNA4 blog posts above) was found for A1111/Forge specifically.
-  Given ComfyUI already has an AMD-blessed path plus a first-class OmniRoute
-  provider (below), there's no reason to take on a community fork's
-  maintenance risk instead.
-
-### InvokeAI — usable but weaker AMD story for a new-generation card
-
-- InvokeAI documents ROCm support but flags it as second-tier: "AMD GPUs
-  are only supported on Linux," and "support for newer AMD GPUs is spotty
-  ... you may experience garbled images, black images, or long startup
-  delays." Its own install docs reference ROCm 5.4.2-era wheels, notably
-  older than the ROCm 7.x this stack's llama-server image already runs on
-  gfx1201.
-  Source: [InvokeAI installation docs (mauwii mirror)](https://mauwii.github.io/InvokeAI/installation/030_INSTALL_CUDA_AND_ROCM/).
-- No OpenAI-compatible-images angle either — same drawback as A1111/Forge.
-  Not recommended as primary given ComfyUI's stronger, more current AMD
-  documentation trail.
-
-## Model choice: FLUX.1 [schnell] vs FLUX.1 [dev] vs SDXL vs SD3.5
-
-| Model | License | Params | Notes |
-|---|---|---|---|
-| **FLUX.1 [schnell]** | **Apache-2.0** — fully open, no commercial restriction | 12B | Distilled for 1-4 step inference (fast); Black Forest Labs' own model card states this license directly |
-| FLUX.1 [dev] | [FLUX.1-dev Non-Commercial License](https://github.com/black-forest-labs/flux/blob/main/model_licenses/LICENSE-FLUX1-dev) | 12B | Non-commercial for the *model/weights*; generated *outputs* are explicitly usable commercially per BFL's license page. Higher quality than schnell (more steps, non-distilled) but the weights themselves can't be redistributed/used commercially |
-| SDXL | CreativeML OpenRAIL++ (permissive, commercial-friendly) | ~3.5B | Older (2023), lower fidelity than FLUX/SD3.5 by current standards, but lowest VRAM footprint and best long-standing tooling maturity |
-| SD3.5 (Large/Medium) | Stability AI Community License — free commercial use under $1M annual revenue, else enterprise license required | 8B / 2.5B | Free for this repo's non-commercial homelab use regardless; template already listed in AMD's own ComfyUI-ROCm doc ("SD3.5 Simple") as a first-party example workflow |
-
-Sources: [black-forest-labs/flux model cards](https://github.com/black-forest-labs/flux/blob/main/model_cards/FLUX.1-dev.md),
-[black-forest-labs/FLUX.1-schnell on Hugging Face](https://huggingface.co/black-forest-labs/FLUX.1-schnell)
-(license: apache-2.0), [FLUX.1-dev LICENSE.md](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md),
-[Stability AI — Introducing Stable Diffusion 3.5](https://stability.ai/news-updates/introducing-stable-diffusion-3-5),
-[stabilityai/stable-diffusion-3.5-large LICENSE.md](https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/LICENSE.md).
-
-**Recommendation: FLUX.1 [schnell].** For a private homelab, license
-enforcement isn't the deciding factor by itself, but schnell's Apache-2.0
-status removes any future ambiguity if outputs or the setup are ever shared
-or repurposed, and its whole design point — good quality in 1-4 sampling
-steps — directly addresses the VRAM/time-slicing constraint below (less
-GPU-resident time per image than a 20-50 step dev/SDXL/SD3.5 run).
-Quantized via `city96/ComfyUI-GGUF` (an actively-referenced, community-
-trusted quantization node — its GGUF Q-quants dequantize to bf16/fp16 at
-runtime, sidestepping the gfx1201 FP8 dequant bug above entirely), FLUX fits
-in a fraction of its fp16 footprint:
-
-| Precision | Approx. VRAM (model only) |
-|---|---|
-| fp16 (baseline) | ~24 GB |
-| fp8 | ~12 GB |
-| GGUF Q5_K_S | ~12-15 GB (practical quality floor) |
-| GGUF Q4_K_S | ~7 GB (quality starts degrading on hands/text below Q4) |
-
-Source: aggregated VRAM figures from GGUF-quantization write-ups referencing
-city96's FLUX GGUF conversions — treat as secondary/community sourced
-(no single BFL-published VRAM table was found), consistent across multiple
-independent sources.
-[city96/ComfyUI-GGUF README](https://github.com/city96/ComfyUI-GGUF/blob/main/README.md),
-[city96/FLUX.1-dev-gguf model card](https://huggingface.co/city96/FLUX.1-dev-gguf).
-
-**Fallback pick: SDXL.** If schnell's distilled quality ceiling proves too
-low for some use case, SDXL is the safer second choice over FLUX.1 [dev] or
-SD3.5 specifically *because* of this card's tight headroom: it's the
-smallest of the four by a wide margin, has the longest production track
-record on ROCm of any of these models, and its OpenRAIL++ license carries no
-revenue-threshold clause to track (unlike SD3.5's Community License) or
-non-commercial weight restriction (unlike FLUX.1 [dev]).
-
-## OpenAI-compatible API / OmniRoute integration
-
-This is the best news in this research: **OmniRoute already ships a
-first-class, built-in `comfyui` provider** — not a generic "point it at an
-OpenAI base URL and hope" integration. Its provider reference documents it
-explicitly: *"No API key required. Configure the local ComfyUI base URL
-(default: http://localhost:8188)."* OmniRoute's own image-routing feature
-set (`/v1/images/generations`, `/v1/images/edits`, `/v1/images/variations`,
-with automatic provider fallback) is designed for exactly this pattern:
-register ComfyUI as a backend, then any client already calling OmniRoute's
-OpenAI-compatible images endpoints reaches it with no extra shim.
-Source: [diegosouzapw/OmniRoute PROVIDER_REFERENCE.md](https://github.com/diegosouzapw/OmniRoute/blob/main/docs/reference/PROVIDER_REFERENCE.md),
-[diegosouzapw/OmniRoute repo description](https://github.com/diegosouzapw/OmniRoute).
-
-This means **ComfyUI does not need an extra OpenAI-API wrapper project** —
-the wrapper projects found during this research
-([`ComfyUI-OpenAI-Compatible-API`](https://github.com/yeeyou/ComfyUI-OpenAI-Compatible-API))
-turned out to be a ComfyUI *custom node* for calling *outbound* to LLM APIs
-from within a workflow (the reverse direction), not something this stack
-needs — OmniRoute's own native ComfyUI provider is the actual integration
-point, one layer up.
-
-**Confidence note:** the provider-reference detail above was fetched via an
-automated summarizer against the raw doc rather than manually re-verified
-line-by-line; re-check `PROVIDER_REFERENCE.md`'s `comfyui` entry directly
-before wiring this up, in case ComfyUI's own `/prompt` API (a
-workflow-graph-shaped API, not a simple text-prompt-in/image-out call) needs
-a specific default workflow JSON configured on the OmniRoute side to produce
-a plain text-to-image call.
-
-## Integration sketch (not a full compose — see caveats above)
-
-- New service in `docker-compose.yml`, e.g. `comfyui`, image
-  `rocm/comfyui-rocm` (or `yurisasc/comfyui-rocm-rdna4` for a gfx1201-tuned
-  build) or built from AMD's own ROCm ComfyUI Dockerfile, same
-  `/dev/kfd` + `/dev/dri` + `group_add: [video, render]` device-passthrough
-  block already used for `llama-server`, joined to the same `ai-stack`
-  network so `omniroute` can reach it as `http://comfyui:8188` — no host
-  port needed (matches the existing llama-server pattern of no published
-  port, gateway-only access).
-- Register it in OmniRoute's dashboard as a `comfyui` provider pointing at
-  that internal URL, same manual-registration pattern already used for
-  llama-server and searxng-search per `docs/proxy-key-onboarding.md`.
-- **VRAM contention is the real design problem, not networking.** Given the
-  ~6 GB headroom, the two services can't both sit GPU-resident.
-  Two workable patterns, in order of how well they fit what's already in
-  this repo:
-  1. **Mirror the existing lazytainer stop-on-idle pattern** already applied
-     to `llama-server` (`docker-compose.yml`'s `lazytainer.group.*` labels) —
-     add an equivalent idle-timeout group for `comfyui`, and rely on the two
-     services naturally not being hit at the same time for a single-user
-     homelab. This doesn't *guarantee* mutual exclusion (both could still be
-     woken concurrently and both try to fit in 6 GB free), so it's a
-     reasonable-effort fit, not a hard guarantee.
-  2. **Explicit mutual exclusion**: a small script/compose profile that
-     stops `llama-server` before starting `comfyui` (and vice versa) rather
-     than relying on lazytainer's independent idle timers — worth doing if
-     the reasonable-effort version above causes a visible OOM in practice.
-  Either way, this is a "pick one, then the other" story, not "run both."
-- Given FLUX.1 [schnell]'s 1-4 step design, a cold-start-and-generate cycle
-  (wake ComfyUI from lazytainer sleep, generate, let it idle back down) is
-  a reasonably good fit for occasional image requests through the same
-  gateway that already does this for llama-server.
-
-## Sources consulted
-
-- [ROCm docs — ComfyUI on ROCm installation](https://rocm.docs.amd.com/projects/comfyui/en/docs-26.04/install/comfyui-install.html)
-- [ROCm blog — ComfyUI on AMD Radeon RX 9000 Series (RDNA4)](https://rocm.blogs.amd.com/artificial-intelligence/comfyui-radeon-9000/README.html)
-- [comfyanonymous/ComfyUI README](https://github.com/comfyanonymous/ComfyUI)
-- [YanWenKun/ComfyUI-Docker](https://github.com/YanWenKun/ComfyUI-Docker)
-- [yurisasc/comfyui-rocm-rdna4](https://github.com/yurisasc/comfyui-rocm-rdna4)
-- [charlie12345/R9700AIProComfyUIPatch](https://github.com/charlie12345/R9700AIProComfyUIPatch)
-- [ROCm/TransformerEngine issue #520 (gfx1201 FP8 fallback)](https://github.com/ROCm/TransformerEngine/issues/520)
-- [lllyasviel/stable-diffusion-webui-forge discussion #67 (AMD support)](https://github.com/lllyasviel/stable-diffusion-webui-forge/discussions/67)
-- [InvokeAI CUDA/ROCm install docs](https://mauwii.github.io/InvokeAI/installation/030_INSTALL_CUDA_AND_ROCM/)
-- [black-forest-labs/flux GitHub repo + model cards](https://github.com/black-forest-labs/flux)
-- [black-forest-labs/FLUX.1-schnell (Hugging Face, Apache-2.0)](https://huggingface.co/black-forest-labs/FLUX.1-schnell)
-- [black-forest-labs/FLUX.1-dev LICENSE.md](https://huggingface.co/black-forest-labs/FLUX.1-dev/blob/main/LICENSE.md)
-- [Stability AI — Introducing Stable Diffusion 3.5](https://stability.ai/news-updates/introducing-stable-diffusion-3-5)
-- [stabilityai/stable-diffusion-3.5-large LICENSE.md](https://huggingface.co/stabilityai/stable-diffusion-3.5-large/blob/main/LICENSE.md)
-- [city96/ComfyUI-GGUF](https://github.com/city96/ComfyUI-GGUF)
-- [city96/FLUX.1-dev-gguf](https://huggingface.co/city96/FLUX.1-dev-gguf)
-- [diegosouzapw/OmniRoute](https://github.com/diegosouzapw/OmniRoute) and its `PROVIDER_REFERENCE.md`
-- [yeeyou/ComfyUI-OpenAI-Compatible-API](https://github.com/yeeyou/ComfyUI-OpenAI-Compatible-API) (checked and ruled out — wrong direction)
-- This repo: `docker-compose.yml`, `.env.example`, `docs/research/qwen3.8-27b-quant.md`

docs/research/inference-engine-spec-decoding-multi-gpu.md

@@ -0,0 +1,144 @@
+# Research: which inference engine for better drafting now and two-GPU splitting later (plus the `--cache-ram` typo bug)
+
+**Date:** 2026-11-02
+
+**Question:** llama.cpp is said to lag on newer drafters (DFlash/DSpark, EAGLE-3/EAGLE-4) and its multi-GPU tensor parallelism is experimental. Which engine fits the server best for speed and efficiency, with a better drafter now and the main model split across two R9700s later? Constraints: commercial use must be allowed (engine and weights), two agents sometimes share the backend, there is an overnight backlog of API calls (throughput matters, not only single-stream latency), and context stays as it is. Second question: why does the main model misspell words when the RAM prompt cache (`--cache-ram`) is on?
+
+**Scope note:** research only. Nothing in compose, `.env.example` or `models.yml` was changed. Verified facts carry a source link. Lines marked **(inference)** are my reasoning, not something a source states. Engines and drafters move weekly in this area, so anything here older than about a month should be re-checked.
+
+## Answer, short version
+
+**Now (one R9700): stay on llama.cpp.** The premise is partly out of date. llama.cpp master has `draft-eagle3`, `draft-dflash`, `draft-dspark` and `draft-mtp` as speculative types ([docs/speculative.md](https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md), [common/speculative.cpp](https://github.com/ggml-org/llama.cpp/blob/master/common/speculative.cpp)), and DFlash2 was merged on 2026-08-27 ([#27342](https://github.com/ggml-org/llama.cpp/pull/27342)). No engine on a single R9700 has a measured result that beats the current setup for this model:
+
+- SGLang does not build for gfx1201 on `main` ([sglang#46900](https://github.com/sgl-project/sglang/issues/46900)).
+- Upstream vLLM lists gfx1201, but nobody has published single-R9700 numbers for Qwen3.8-27B. Its official FP8 checkpoint (~28.5 GB, per the [SGLang cookbook](https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.8-27B)) doesn't fit beside a usable KV cache on 32 GB.
+- EAGLE-4 isn't in llama.cpp and its license limits commercial use (below).
+
+Cheap wins on llama.cpp:
+
+1. **Keep `LLAMA_CACHE_RAM=0`.** That is the workaround for the typo bug until [#27624](https://github.com/ggml-org/llama.cpp/pull/27624) merges (diagnosis below).
+2. A/B **`draft-dspark` with `RedHatAI/Qwen3.8-27B-speculator.dspark`** (Apache-2.0) against DFlash2. It is an option only: no GGUF of it exists yet, and conversion is unverified for this target.
+3. Watch two open, R9700-specific llama.cpp performance issues. Both hit exactly this workload, speculative verify batches with two slots: [#28867](https://github.com/ggml-org/llama.cpp/issues/28867) (~20 % spec-decode loss from a flash-attention dispatch threshold) and [#29820](https://github.com/ggml-org/llama.cpp/issues/29820) (up to +44 % from an MMVQ batch-size change).
+
+**When GPU #2 arrives: move the main model to vLLM with tensor parallel 2.** The only real-world TP2 evidence on 2× R9700 for this exact model is vLLM. One publicly documented setup serves `Qwen/Qwen3.8-27B-FP8` with MTP-3 at 262k context and reaches 66 tok/s single-stream and ~1,014 tok/s aggregate at 32 concurrent sequences ([ROCm/vllm#1229](https://github.com/ROCm/vllm/issues/1229)). That is the batch throughput the overnight backlog needs. llama.cpp's `--split-mode tensor` is labelled EXPERIMENTAL in its own help text ([common/arg.cpp](https://github.com/ggml-org/llama.cpp/blob/master/common/arg.cpp)), and it crashes with DFlash drafters today ([#27833](https://github.com/ggml-org/llama.cpp/issues/27833), [#28777](https://github.com/ggml-org/llama.cpp/issues/28777)). Use MTP-3 (built into the checkpoint) as the default, because it scales with concurrency. DFlash2 wins only at 1-4 users ([RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)). Every engine and weight file in that path is MIT or Apache-2.0.
+
+**Avoid for work use:** EAGLE-4 (source-available, commercial use only below USD 20M aggregate revenue of the licensee and its affiliates, [LICENSE](https://github.com/eagle-hq/EAGLE-4/blob/main/LICENSE)) and the `RadixArk/Qwen3.8-27B-DSpark` drafter (license `other`, no terms in the card, [model card](https://huggingface.co/RadixArk/Qwen3.8-27B-DSpark)).
+
+## 1. Current state (from this repo)
+
+`docker-compose.yml` → `llama-server`: `ghcr.io/ggml-org/llama.cpp:server-rocm`, `--parallel 2`, `--kv-unified`, `--flash-attn on`, `--cache-type-k/v ${LLAMA_KV_CACHE_TYPE}`, `--cache-ram ${LLAMA_CACHE_RAM}`, `${LLAMA_SPEC_ARGS}`, `--jinja`. `.env.example` sets `LLAMA_CACHE_RAM=0` and `LLAMA_SPEC_ARGS` to DFlash2 (Q8_0 drafter, `--spec-draft-n-max 7`). Earlier measurements in [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md) found 45.9 tok/s weighted decode, 50 % draft acceptance, and prefill of ~950 tok/s at 40-50k tokens.
+
+Note: compose's fallback for `--cache-ram` is still `24576`, while `.env.example` says `0`. The server's `.env` wins, but if the key were ever missing, the cache would silently come back on. Not changed here.
+
+Model facts that drive everything below, from [Qwen/Qwen3.8-27B config.json](https://huggingface.co/Qwen/Qwen3.8-27B/raw/main/config.json):
+
+- 64 layers with `full_attention_interval: 4`, so 16 attention layers and 48 Gated DeltaNet (recurrent) layers.
+- `mtp_num_hidden_layers: 1`, so the official weights **ship an MTP head**.
+- License Apache-2.0 ([model card](https://huggingface.co/Qwen/Qwen3.8-27B)).
+
+## 2. Comparison table
+
+| Engine | RDNA4 / gfx1201 (R9700) | Spec decoding for Qwen3.8-27B | Multi-GPU | Prefix cache (hybrid model) | Batch throughput | Engine license |
+|---|---|---|---|---|---|---|
+| **llama.cpp** (current) | Official ROCm and Vulkan builds. Open R9700 correctness/perf issues: [#30884](https://github.com/ggml-org/llama.cpp/issues/30884) (FA long-context EOS on Windows), [#28867](https://github.com/ggml-org/llama.cpp/issues/28867), [#29820](https://github.com/ggml-org/llama.cpp/issues/29820) | `draft-mtp`, `draft-dflash` (DFlash2), `draft-dspark`, `draft-eagle3`, n-gram types ([docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md)). No EAGLE-4. No Qwen3.8-27B EAGLE-3 drafter found on HF | `-sm layer` (default, pipelined: more VRAM, no single-stream speed-up), `row`, `tensor` (EXPERIMENTAL). DFlash + `-sm tensor` crashes ([#27833](https://github.com/ggml-org/llama.cpp/issues/27833), [#28777](https://github.com/ggml-org/llama.cpp/issues/28777)). Repeated-garbage bug on CUDA ([#30075](https://github.com/ggml-org/llama.cpp/issues/30075)) | Per-slot prompt plus context checkpoints, plus `--cache-ram` RAM tier, with open correctness bugs (§4) | Fixed slots. Context is divided per slot. DFlash acceptance collapses at high slot counts ([#27117](https://github.com/ggml-org/llama.cpp/issues/27117)) | MIT ([LICENSE](https://github.com/ggml-org/llama.cpp/blob/master/LICENSE)) |
+| **vLLM** (upstream) | Listed: "Radeon RX 9000 series (gfx1200/1201)", ROCm image built for gfx1201 ([install docs](https://docs.vllm.ai/en/latest/getting_started/installation/gpu/), [Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base)). AITER and LMCache are built for gfx942/950 only (same Dockerfile). No CI pool covers gfx11/gfx12 ([#64614](https://github.com/vllm-project/vllm/pull/64614)) | `mtp`/`qwen3_5_mtp`, `dflash`, `dspark`, `eagle`, `eagle3`, `draft_model`, `ngram`, `suffix` ([speculative.py](https://github.com/vllm-project/vllm/blob/main/vllm/config/speculative.py)). Recipe: DFlash2 needs ≥0.28.0 ([recipe](https://recipes.vllm.ai/Qwen/Qwen3.8-27B)). Stock vLLM fails to load an FP8 DFlash drafter ([RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)) | Tensor parallel, the mature path. TP2 on 2× R9700 in production ([ROCm/vllm#1229](https://github.com/ROCm/vllm/issues/1229)). `NCCL_PROTO=Simple` recommended for PCIe | Hybrid-aware: `mamba_cache_mode="align"` caches recurrent state at block boundaries ([cache.py](https://github.com/vllm-project/vllm/blob/main/vllm/config/cache.py)). Optional CPU KV offload (`kv_offloading_size`, same file) | Paged KV shared by all sequences, continuous batching. 1,014 tok/s aggregate at c32 on 2× R9700 ([#1229](https://github.com/ROCm/vllm/issues/1229)) | Apache-2.0 ([LICENSE](https://github.com/vllm-project/vllm/blob/main/LICENSE)) |
+| **vLLM RDNA4 fork** ([Capicua25x/vllm-rocm-rdna4](https://github.com/Capicua25x/vllm-rocm-rdna4)) | Built for gfx1200/1201. Validated on gfx1201 TP2 only. Based on 0.28.0 (rc13, 2026-09-03), while upstream is at v0.33.0rc3 (2026-11-02) | MTP-3 (default), DFlash2 including FP8 drafters (fix `50f7a7d12`) | TP2 measured | `align` mode, fp8 KV plus bf16 SSM state | MTP-3: 315 agg @c16 on 6k prompts. DFlash2 is ahead only to ~c4 | Apache-2.0 (one-person fork) |
+| **SGLang** | **Not supported on `main`**. RDNA fixes go to the `amd_march` branch until RDNA CI exists. As of 2026-10-30 `main` fails to build sgl-kernel for gfx1201 ([#46900](https://github.com/sgl-project/sglang/issues/46900), [#30599](https://github.com/sgl-project/sglang/issues/30599)) | EAGLE/NEXTN (MTP), DFLASH, DSPARK ([cookbook](https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.8-27B)). EAGLE-4 via `eagle-harness` (CUDA B300, MI355X only) | TP mature on CUDA/CDNA | Radix cache with mamba strategies (cookbook) | Strong on supported hardware | Apache-2.0 ([LICENSE](https://github.com/sgl-project/sglang/blob/main/LICENSE)) |
+| TGI | n/a | n/a | n/a | n/a | n/a | Repo archived, "maintenance mode", recommends vLLM/SGLang/llama.cpp ([README](https://github.com/huggingface/text-generation-inference)) |
+| ExLlamaV3 / TabbyAPI | CUDA only (install requires CUDA ≥12.4, [README](https://github.com/turboderp-org/exllamav3)) | n/a | n/a | n/a | n/a | MIT |
+
+Lemonade ([lemonade-sdk/lemonade](https://github.com/lemonade-sdk/lemonade), Apache-2.0) and LM Studio/Ollama wrap llama.cpp for GGUF. They add no drafter or tensor-parallel capability beyond llama.cpp, so they are not evaluated further **(inference from their being llama.cpp front-ends)**.
+
+## 3. Drafters and weights: what exists, and can it be used at work?
+
+| Artifact | What | Engine support | License |
+|---|---|---|---|
+| `Qwen/Qwen3.8-27B` | Target, BF16, MTP head included | all | Apache-2.0 ([HF](https://huggingface.co/Qwen/Qwen3.8-27B)) |
+| `Qwen/Qwen3.8-27B-FP8` | Official FP8, ~28.5 GB | vLLM, SGLang | Apache-2.0 ([HF](https://huggingface.co/Qwen/Qwen3.8-27B-FP8)) |
+| `unsloth/Qwen3.8-27B-GGUF` | Current GGUF, plus `MTP/mtp-Qwen3.8-27B-Q4_0.gguf` | llama.cpp | Apache-2.0 ([HF](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF)) |
+| `amd/Qwen3.8-27B-Quark-AWQ-INT4-W4A16`, `RedHatAI/Qwen3.8-27B-INT4` | INT4 W4A16, ~19.5 GB each | vLLM (RDNA hybrid W4A16 kernel, below) | Apache-2.0 ([AMD](https://huggingface.co/amd/Qwen3.8-27B-Quark-AWQ-INT4-W4A16), [RedHat](https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4)) |
+| `z-lab/Qwen3.8-27B-DFlash2` (+ `-GGUF`) | DFlash2 drafter, 3.85 GB bf16, the current one | llama.cpp, vLLM, SGLang | Apache-2.0 ([HF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2), [GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF)) |
+| `incoai/Qwen3.8-27B-DFlash2` | DFlash2, used in the vLLM and SGLang recipes | vLLM, SGLang | Apache-2.0 ([HF](https://huggingface.co/incoai/Qwen3.8-27B-DFlash2)) |
+| `RedHatAI/Qwen3.8-27B-speculator.dspark` | DSpark, block 8, confidence head, 3.98 GB | vLLM (card). llama.cpp converts speculators-format DSpark ([docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md)), but this target is **untested** | Apache-2.0 ([HF](https://huggingface.co/RedHatAI/Qwen3.8-27B-speculator.dspark)) |
+| `RadixArk/Qwen3.8-27B-DSpark` | "DSpark v2", 1.86B | SGLang | `other`, no terms in card: **do not use until clarified** ([HF](https://huggingface.co/RadixArk/Qwen3.8-27B-DSpark)) |
+| `EAGLE-Corp/Qwen3.8-27B-EAGLE4` | EAGLE-4. Claims 3.59× at T=0 vs 2.80× DFlash2 (4× B300, c1) | `eagle-harness` for SGLang/vLLM/TokenSpeed on B300 and MI355X ([README](https://github.com/eagle-hq/EAGLE-4)). Not llama.cpp, not RDNA | Source-available, commercial use only while licensee plus affiliates are under USD 20M revenue per 12 months, and no "Competing Offering" ([LICENSE §3](https://github.com/eagle-hq/EAGLE-4/blob/main/LICENSE)). **Not suitable for work use without legal review** |
+
+Speculation gains shrink with concurrency for every method. EAGLE-4's own table drops from 3.42× at 1 request to 1.39× at 64 on SGLang ([README](https://github.com/eagle-hq/EAGLE-4)). On 2× R9700, DFlash2 gives +18-26 % single-stream over MTP-3 but falls behind it from c8 up ([RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)). For the overnight backlog, the drafter matters less than paged batching.
+
+## 4. Diagnosis: misspellings with `--cache-ram` on
+
+### What's verified upstream
+
+1. **[#27148](https://github.com/ggml-org/llama.cpp/issues/27148) (open): the RAM prompt cache restores state from an unrelated conversation into a slot.** When `get_available_slot()` falls back to an LRU victim, `server_prompt_cache::load()` returns `true` even when nothing matched. `prompt_clear()` then never runs, so the victim slot keeps its previous owner's KV and recurrent state. The issue has independent reproductions:
+   - on Qwen3.8-27B (hybrid GDN plus attention) with `--kv-unified` on Metal, where responses "switched mid-sentence into another request's output"
+   - with strictly sequential single-user requests on gfx1151 ROCm with `kv_unified = true`
+
+   Mitigation from the thread: with `kv_unified = true`, `--cache-ram 0` **or** `--no-cache-idle-slots` closes it. With `kv_unified = false`, both are needed.
+2. **Fix PR [#27624](https://github.com/ggml-org/llama.cpp/pull/27624) (open):** it replaces the boolean with `RESTORED`/`NO_MATCH`/`FAILED`, and clears an LRU victim when restore fails, nothing matches, or `cache_ram=0`. The PR says the `cache_ram=0` path has the same missing clear.
+3. **Related hardening [#28058](https://github.com/ggml-org/llama.cpp/pull/28058) (open):** `llama_memory_recurrent::seq_rm` returned `true` for a partial erase it cannot do, "leaving the removed tokens in the state", so "stale recurrent state leaked into the new prompt positions on hybrid/recurrent models". The PR's main fix targets integrated GPUs and doesn't apply to the R9700. The `seq_rm` part applies to any hybrid model.
+4. **[#30961](https://github.com/ggml-org/llama.cpp/issues/30961) / [#30997](https://github.com/ggml-org/llama.cpp/pull/30997) (both open):** with `--kv-unified` plus `--cache-ram` plus ≥2 slots, a cached prompt fails to restore while another idle slot still holds its cells. That causes a full re-prefill (performance, not corruption). Same config as this stack.
+5. Merged related work, so the build date matters: [#30498](https://github.com/ggml-org/llama.cpp/pull/30498), [#30567](https://github.com/ggml-org/llama.cpp/pull/30567) and [#30574](https://github.com/ggml-org/llama.cpp/pull/30574) (2026-10-15, cache bookkeeping and saving the draft state). [#26004](https://github.com/ggml-org/llama.cpp/pull/26004) (2026-10-08, checkpoints across slot save/restore). None of them fixes #27148.
+
+### Most likely cause here (inference)
+
+The stack runs exactly the affected configuration: hybrid model, `--parallel 2`, `--kv-unified`, prompt cache on. On a pure-attention model, a stale slot shows up as "answers the wrong conversation". On this model, 48 of 64 layers carry a recurrent state that cannot be partially rolled back. Leftover recurrent state from the previous owner, mixed with correct attention KV for the new prompt, would plausibly give output that stays on topic but is subtly corrupted at the token level. That matches "misspells a lot" better than a wholesale swap.
+
+Two things argue against the alternatives:
+
+- **The drafter is not the cause.** The target verifies every draft token, so a stale or mismatched draft state only lowers acceptance ([#30574](https://github.com/ggml-org/llama.cpp/pull/30574) shows exactly that: acceptance dropped, "the output is identical"). Speculative verification on GDN targets is not bit-identical to plain decoding ([#31472](https://github.com/ggml-org/llama.cpp/issues/31472)), but that shows up as paraphrase-level drift with or without the RAM cache.
+- **Quantized KV alone is unlikely.** The symptom tracks `--cache-ram` on/off at a fixed `q8_0`.
+
+Rule out before blaming llama.cpp: in a similar "dropped characters with prompt cache" report on a hybrid Qwen, the root cause was the client's SSE parser stripping repeated deltas ([#21681](https://github.com/ggml-org/llama.cpp/issues/21681)). Check that the typos are present in the raw stream from `llama-server`, not just in what the gateway or Coding CLI shows.
+
+### Workaround and how to confirm
+
+- **Workaround (already in `.env.example`): `LLAMA_CACHE_RAM=0`.** Per the #27148 thread it is sufficient with `--kv-unified`. The cost is the re-prefill that the RAM cache used to save, ~47 s per 45k-token prompt per [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md). Optionally add `--no-cache-idle-slots` as a second guard. It's harmless, since idle-slot caching needs the RAM cache anyway.
+- **Confirm the diagnosis (cheap):** reuse the #27148 / #27624 test shape. Send A (a short unique conversation), then B (an unrelated long prompt with a nonce), then C (A again with a new message), with `--cache-ram` on, `temperature 0`, and the drafter off. Compare C's output against a fresh-server cold run of the same prompt. A diff with cache on and none with cache off confirms it. Log lines to grep: `selected slot by LRU`, `prompt_load`, `failed to restore`.
+- **Re-enable the cache** only on a build that contains #27624 (and ideally #30997), and after the test above passes.
+
+## 5. Single GPU: why not switch to vLLM now?
+
+- **Quant formats.** vLLM's docs table marks AWQ and GPTQ as ❌ on "AMD GPU" ([quantization README](https://github.com/vllm-project/vllm/blob/main/docs/features/quantization/README.md)). That table is stale for RDNA: `HybridW4A16LinearKernel` for gfx11/gfx12 (AWQ, GPTQ, compressed-tensors INT4, group 32/64/128) merged on 2026-07-14 ([#40977](https://github.com/vllm-project/vllm/pull/40977)), with follow-up fixes still landing ([#64614](https://github.com/vllm-project/vllm/pull/64614) merged 2026-11-02, [#52637](https://github.com/vllm-project/vllm/pull/52637) open). GGUF in vLLM is "highly experimental and under-optimized" and moved to an out-of-tree plugin ([gguf.md](https://github.com/vllm-project/vllm/blob/main/docs/features/quantization/gguf.md)).
+- **No measurement exists.** I found no published single-R9700 vLLM numbers for Qwen3.8-27B INT4. The fork documents TP2 only.
+- **VRAM (inference, unmeasured).** INT4 weights are ~19.5 GB. Attention KV is 16 layers × 2 × 4 KV heads × 256 × 1 byte = 32 KiB/token at fp8. With MTP (no separate drafter) and ~0.92 memory utilisation, that leaves roughly 7 GB, about 230k tokens of KV shared across all sequences. A 3.85 GB DFlash2 drafter would cut that to roughly 120k. Today's 262k at `q8_0` would not clearly fit, and speed versus llama.cpp's 46 tok/s is unknown.
+- **ROCm-specific instability.** A vLLM 0.20 → 0.23 image upgrade halved FP8 throughput on 4× R9700, root cause unknown ([Level1Techs](https://forum.level1techs.com/t/vllm-0-23-x-rocm-7-14-upgrade-on-4x-radeon-ai-pro-r9700-50-regression-on-one-model-unaffected-on-another-but-a-free-9-via-kernel-tuning/253460)). AITER generated a kernel over the LDS limit on gfx1201 ([#48723](https://github.com/vllm-project/vllm/issues/48723), open). FP8 RowWise scaled MM is being disabled on RDNA4 ([#57861](https://github.com/vllm-project/vllm/pull/57861), open). A Quark `exclude` bug silently breaks MTP ([#65034](https://github.com/vllm-project/vllm/issues/65034), open).
+
+Verdict: a vLLM trial on one card is a reasonable side experiment (`RedHatAI/Qwen3.8-27B-INT4` plus `--speculative-config '{"method":"mtp","num_speculative_tokens":3}'`, as on [its card](https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4)), but not a replacement until it is benchmarked with `speed-bench` against the current numbers.
+
+## 6. Two GPUs: the plan
+
+1. Same backend on both cards. Both are R9700s on ROCm, as already noted in [`500k-context-2-parallel-agents-gpu-roadmap.md`](500k-context-2-parallel-agents-gpu-roadmap.md).
+2. vLLM TP2, `Qwen/Qwen3.8-27B-FP8` (or the INT4 checkpoint for more KV), `--enable-prefix-caching` (hybrid `align` mode), fp8 KV, MTP-3, `--tool-call-parser qwen3_coder --reasoning-parser qwen3` ([SGLang cookbook](https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.8-27B) names the same parsers, and [RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md) warns that a hermes parser "craters agentic scores"), and `NCCL_PROTO=Simple`.
+3. Start on upstream `vllm/vllm-openai-rocm`. Fall back to the fork's image only if upstream underperforms. The fork is a single maintainer, five minor versions behind, and its FP8-drafter fix isn't upstream.
+4. The overnight backlog benefits most: one shared KV pool, `--max-num-seqs` well above 2, and no per-slot context division. The two-agents-by-day case keeps the large window, because vLLM reserves KV per token used, not per slot.
+5. If llama.cpp's `-sm tensor` stabilises with DFlash (watch #27833, #28777), re-evaluate. Until then, llama.cpp on two cards means `-sm layer`, which buys VRAM (a bigger quant, more slots) but no single-stream speed.
+
+## 7. Migration risks (llama.cpp → vLLM)
+
+- **Weights and downloader.** vLLM loads a safetensors directory (18+ shards), not one GGUF. `models.yml`'s one-file-per-URL downloader would need many entries or a different fetch method. The HF cache volume grows by ~20-30 GB.
+- **Tool-call and reasoning parsing** change completely (vLLM parsers instead of llama.cpp's Jinja/PEG). Re-run the checks from [`qwen3.8-27b-tool-calling.md`](qwen3.8-27b-tool-calling.md) through the gateway. vLLM ignores OpenAI `reasoning_effort` for this family. Thinking is a `chat_template_kwargs` switch ([RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)).
+- **Startup and reproducibility.** First start compiles and autotunes kernels (minutes), and the result depends on timing: different compile caches give different temperature-0 outputs. Persist the compile-cache volume ([RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)).
+- **Gateway timeouts and slot assumptions.** OmniRoute prompt-cache affinity, the per-slot-context rule in [`architecture.md`](../architecture.md) and the Coding CLI context settings are all llama.cpp-specific. They need rewriting, not porting.
+- **ROCm regressions between image tags** (Level1Techs report above). Pin image digests and benchmark each bump.
+- **`GPU_MAX_HW_QUEUES` and numeric `group_add`** ([`architecture.md` → GPU access](../architecture.md#gpu-access)) still apply to any ROCm container. vLLM also needs `--ipc=host` or a large `shm_size` for TP workers ([fork run commands](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)).
+- **Rollback is cheap** if the llama.cpp service stays in compose under a profile.
+
+## 8. Open questions
+
+- Does the server's current `server-rocm` image already contain any of #30498/#30567 (2026-10-15)? `docker logs llama-server | head` shows the build.
+- Is DSpark conversion (`convert_hf_to_gguf.py --target-model-dir`) supported for a `Qwen3_5ForConditionalGeneration` verifier? The docs say "only drafts with a Qwen3 backbone". The RedHat draft's layers are Qwen3-style, but this is untested.
+- Single-R9700 vLLM INT4 numbers: decode, prefill at 40k, and the KV pool size printed at boot.
+- RadixArk DSpark license terms (ask the publisher).
+
+## Sources
+
+- Repo: [`docker-compose.yml`](../../docker-compose.yml), [`.env.example`](../../.env.example), [`models.yml`](../../models.yml), [`architecture.md`](../architecture.md), [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md), [`500k-context-2-parallel-agents-gpu-roadmap.md`](500k-context-2-parallel-agents-gpu-roadmap.md), [`qwen3.8-27b-tool-calling.md`](qwen3.8-27b-tool-calling.md)
+- llama.cpp: [LICENSE](https://github.com/ggml-org/llama.cpp/blob/master/LICENSE), [docs/speculative.md](https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md), [common/arg.cpp](https://github.com/ggml-org/llama.cpp/blob/master/common/arg.cpp), [common/speculative.cpp](https://github.com/ggml-org/llama.cpp/blob/master/common/speculative.cpp), PRs [#27342](https://github.com/ggml-org/llama.cpp/pull/27342), [#27624](https://github.com/ggml-org/llama.cpp/pull/27624), [#28058](https://github.com/ggml-org/llama.cpp/pull/28058), [#30997](https://github.com/ggml-org/llama.cpp/pull/30997), [#30498](https://github.com/ggml-org/llama.cpp/pull/30498), [#30567](https://github.com/ggml-org/llama.cpp/pull/30567), [#30574](https://github.com/ggml-org/llama.cpp/pull/30574), [#26004](https://github.com/ggml-org/llama.cpp/pull/26004), [#25051](https://github.com/ggml-org/llama.cpp/pull/25051); issues [#27148](https://github.com/ggml-org/llama.cpp/issues/27148), [#30961](https://github.com/ggml-org/llama.cpp/issues/30961), [#21681](https://github.com/ggml-org/llama.cpp/issues/21681), [#31472](https://github.com/ggml-org/llama.cpp/issues/31472), [#27117](https://github.com/ggml-org/llama.cpp/issues/27117), [#27833](https://github.com/ggml-org/llama.cpp/issues/27833), [#28777](https://github.com/ggml-org/llama.cpp/issues/28777), [#30075](https://github.com/ggml-org/llama.cpp/issues/30075), [#28867](https://github.com/ggml-org/llama.cpp/issues/28867), [#29820](https://github.com/ggml-org/llama.cpp/issues/29820), [#30884](https://github.com/ggml-org/llama.cpp/issues/30884)
+- vLLM: [GPU install docs](https://docs.vllm.ai/en/latest/getting_started/installation/gpu/) (page dated 2026-10-08), [Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base), [quantization README](https://github.com/vllm-project/vllm/blob/main/docs/features/quantization/README.md), [gguf.md](https://github.com/vllm-project/vllm/blob/main/docs/features/quantization/gguf.md), [config/cache.py](https://github.com/vllm-project/vllm/blob/main/vllm/config/cache.py), [config/speculative.py](https://github.com/vllm-project/vllm/blob/main/vllm/config/speculative.py), [LICENSE](https://github.com/vllm-project/vllm/blob/main/LICENSE), [Qwen3.8-27B recipe](https://recipes.vllm.ai/Qwen/Qwen3.8-27B), PRs/issues [#40977](https://github.com/vllm-project/vllm/pull/40977), [#64614](https://github.com/vllm-project/vllm/pull/64614), [#52637](https://github.com/vllm-project/vllm/pull/52637), [#57861](https://github.com/vllm-project/vllm/pull/57861), [#48723](https://github.com/vllm-project/vllm/issues/48723), [#65034](https://github.com/vllm-project/vllm/issues/65034), [ROCm/vllm#1229](https://github.com/ROCm/vllm/issues/1229)
+- vLLM RDNA4 fork: [repo](https://github.com/Capicua25x/vllm-rocm-rdna4), [RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)
+- SGLang: [#30599](https://github.com/sgl-project/sglang/issues/30599), [#46900](https://github.com/sgl-project/sglang/issues/46900), [Qwen3.8-27B cookbook](https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.8-27B), [LICENSE](https://github.com/sgl-project/sglang/blob/main/LICENSE)
+- EAGLE-4: [repo README](https://github.com/eagle-hq/EAGLE-4), [LICENSE](https://github.com/eagle-hq/EAGLE-4/blob/main/LICENSE), [EAGLE-Corp/Qwen3.8-27B-EAGLE4](https://huggingface.co/EAGLE-Corp/Qwen3.8-27B-EAGLE4)
+- Weights: [Qwen/Qwen3.8-27B](https://huggingface.co/Qwen/Qwen3.8-27B) and [config.json](https://huggingface.co/Qwen/Qwen3.8-27B/raw/main/config.json), [Qwen/Qwen3.8-27B-FP8](https://huggingface.co/Qwen/Qwen3.8-27B-FP8), [unsloth GGUF](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF), [z-lab DFlash2](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2), [z-lab DFlash2 GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF), [incoai DFlash2](https://huggingface.co/incoai/Qwen3.8-27B-DFlash2), [RedHatAI DSpark](https://huggingface.co/RedHatAI/Qwen3.8-27B-speculator.dspark), [RadixArk DSpark](https://huggingface.co/RadixArk/Qwen3.8-27B-DSpark), [RedHatAI INT4](https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4), [AMD Quark INT4](https://huggingface.co/amd/Qwen3.8-27B-Quark-AWQ-INT4-W4A16)
+- Others: [TGI](https://github.com/huggingface/text-generation-inference), [ExLlamaV3](https://github.com/turboderp-org/exllamav3), [Lemonade](https://github.com/lemonade-sdk/lemonade), [Level1Techs vLLM ROCm regression thread](https://forum.level1techs.com/t/vllm-0-23-x-rocm-7-14-upgrade-on-4x-radeon-ai-pro-r9700-50-regression-on-one-model-unaffected-on-another-but-a-free-9-via-kernel-tuning/253460)
+
+Confidence: high for licenses, flag and spec-type lists, and issue/PR states (read via the GitHub and HF APIs on 2026-11-02). Medium for the third-party throughput figures (self-reported, not re-run). Low for the single-card vLLM VRAM estimate and the hybrid-state explanation of the typos, which are both marked as inference.

docs/research/omniroute-direct-response-timeout-outage-2026-09-15.md

@@ -0,0 +1,397 @@
+# A 48-minute total outage on `qwen3.8-27b-local`, and the third OmniRoute timeout mechanism this repo hadn't documented yet
+
+**Date:** 2026-09-15
+
+**Verdict:** The error — `"[504]: Direct response did not start within 30000ms — retrying on a fresh socket"` —
+comes from a **third, previously-undocumented OmniRoute timeout mechanism** (`OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS`,
+default 30s), distinct from both timeouts already recorded in
+[`omniroute-account-semaphore-timeout.md`](./omniroute-account-semaphore-timeout.md) and
+[`omniroute-non-ping-sse-stream-timeout.md`](./omniroute-non-ping-sse-stream-timeout.md). It exists specifically to
+recover from a *stale pooled TCP socket* by retrying once on a brand-new connection — but in the incident analyzed
+here, **both** the original attempt and the fresh-socket retry timed out, repeatedly, for 46 requests over 48
+straight minutes with zero successes. That pattern rules out a stale-socket explanation (a fresh socket bypasses
+the pool entirely) and points instead at the upstream itself — `llama-server`, or the R9700 GPU underneath it —
+being genuinely unresponsive for the whole window. The best primary-source match for that symptom is an **open,
+still-unresolved AMD ROCm bug specific to this exact GPU** ([ROCm/legacy-rocm-build#6630](https://github.com/ROCm/legacy-rocm-build/issues/6630)):
+an MES-firmware hang during generation on `gfx1201`/R9700 that leaves the process alive but stuck, sometimes for
+no logged reason at all. No config change fixes this — raising the 30s timeout only makes each failed attempt
+take longer to give up, it doesn't un-wedge a hung GPU.
+
+## The evidence
+
+**Source:** `omniroute-request-logs-6h-2026-09-15.json`, a 339-entry OmniRoute request-log export the user pulled
+from the dashboard, covering `2026-09-15T12:49:52Z`–`18:44:54Z`. Every entry with a non-200 status (66 of them)
+is a `POST /v1/chat/completions` against `qwen3.8-27b-local` (`/models/Qwen3.8-27B-UD-Q4_K_XL.gguf`), all on the
+same `connectionId` (`649a2d3e-7527-488e-9b8a-dc4ac2624176`) and the same `provider`
+(`openai-compatible-chat-a7bda643-6687-41f4-b75d-fd2cab746874`) — a single upstream connection, not a fan-out
+artifact.
+
+Sorting every error by timestamp shows **one continuous outage**, not scattered slow requests:
+
+- Last successful `/v1/chat/completions` before the outage: `15:18:12.285Z`
+- **First failure:** `15:25:06.415Z` — status 504, `"[504]: Direct response did not start within 30000ms —
+  retrying on a fresh socket"`, duration `60186ms`
+- **Every single `/v1/chat/completions` attempt** from `15:25:06.415Z` through `16:13:29.502Z` failed — 46× 504
+  (all clustered `60021`-`60324ms`, i.e. two back-to-back 30s attempts, both failing) interleaved with 20× 499
+  (`"Request aborted"` / `"Client disconnected: request_signal_aborted"`, durations `1.4s`-`99.97s` — these are
+  qwen-code giving up client-side while OmniRoute was still mid-retry)
+- **First successful recovery:** `16:13:50.388Z`, `20873ms` — 21 minutes after the last failure attempt cluster,
+  i.e. the very next attempt after the outage window succeeded normally
+- No successful `/v1/chat/completions` call appears anywhere inside the `15:25:06Z`-`16:13:29Z` window — confirmed
+  by filtering all 154 `/v1/chat/completions` log entries in that range: every one is 504 or 499.
+
+`git log --since=2026-09-14 --until=2026-09-16` shows **zero commits** in this repo on 2026-09-15 — the outage
+correlates with no deploy, `scripts/update.sh` run, or config change on this end.
+
+**A second, independent data point** (see caveat below): the user also pasted a large raw text table, copied
+directly from OmniRoute's dashboard UI rather than the JSON export, showing the **`qwen-classifier`** connection
+(`qwen3-4b`, both the `-UD-Q4_K_XL.gguf` file this repo's `docker-compose.yml` currently defaults to, and a
+`-UD-Q8_K_XL.gguf` variant that appears **nowhere** in this repo's checked-in `docker-compose.yml`/`.env.example`
+— either tested by hand against `LLAMA_CLASSIFIER_MODEL_FILE` outside version control, or evidence of drift worth
+checking directly on the server) failing repeatedly across two accounts (`Haylan`, `qwen-cli-main`), with the
+same signature: `TI: 0|TO: 0` (zero tokens either direction — failed before generating anything) and durations
+clustering at exactly `60.0`-`60.4s` for the 504s. That's the identical two-attempts-at-30s-each shape as the 27B
+outage above, strongly suggesting the same underlying mechanism, though — important caveat — **this table is not
+present in the 6-hour JSON export and covers a different, longer time range** (timestamps back to `01:21` and
+`23:54` on unspecified dates), so it cannot be directly time-correlated against the 27B outage above. Treat it as
+corroborating evidence that this failure mode recurs on both local model connections, not as proof they failed at
+the same moment.
+
+## Root cause: `OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS`, a mechanism built for a different problem
+
+Confirmed directly against OmniRoute's own source, [diegosouzapw/OmniRoute](https://github.com/diegosouzapw/OmniRoute)
+(same repo the two existing timeout docs already cite) — `open-sse/utils/directResponseStartTimeout.ts`:
+
+```ts
+const DEFAULT_DIRECT_HEADERS_TIMEOUT_MS = 30_000;
+const DIRECT_RESPONSE_START_TIMEOUT_CODE = "DIRECT_RESPONSE_START_TIMEOUT";
+
+export function resolveDirectHeadersTimeoutMs(
+  env: Record<string, string | undefined> = process.env
+): number {
+  const raw = env.OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS;
+  if (raw == null || raw.trim() === "") return DEFAULT_DIRECT_HEADERS_TIMEOUT_MS;
+  ...
+}
+
+function createDirectResponseStartTimeout(timeoutMs: number): Error & { code: string } {
+  const err = new Error(
+    `Direct response did not start within ${timeoutMs}ms — retrying on a fresh socket`
+  ) as Error & { code: string };
+  ...
+}
+```
+
+"Direct" here means **direct (no-proxy) egress** — confirmed in `open-sse/utils/proxyFetch.ts`, which routes any
+connection with no configured upstream HTTP proxy through this path (as opposed to OmniRoute's separate
+proxy/relay egress paths). Every local connection in this repo (`llama-server`, `qwen-classifier`, both reached
+over the `ai-stack` Docker network with no proxy) is "direct" — so this timeout mechanism governs **every**
+request to either local model, streaming or non-streaming alike, not just non-streaming JSON responses as the
+name might suggest.
+
+### Why it exists (and why it didn't help here)
+
+Two OmniRoute issues, both with dedicated regression tests in the repo, explain the actual design intent:
+
+- **#4252** (`tests/unit/proxyfetch-retry-fresh-socket-4252.test.ts`): "Undici dispatcher fails on direct provider
+  requests in 502 bursts" — the default direct dispatcher pools keep-alive sockets; some upstreams silently close
+  idle pooled sockets, so the next request reusing one fails with `UND_ERR_SOCKET`. Fix: retry once on a **fresh,
+  no-keep-alive dispatcher** (`getRetryDispatcher()`, a different instance from `getDefaultDispatcher()`) so the
+  retry can't grab another already-dead pooled socket.
+- **#10214** (`tests/unit/proxyfetch-direct-response-start-timeout-10214.test.ts`): "Direct (no-proxy) requests
+  stall on a silently-dropped pooled keep-alive socket until the caller's deadline or a service restart" — the
+  harder case: a pooled socket that dies **without even an error**, just silence. Undici's `headersTimeout`
+  default (600s) is far too slow to catch this in practice, and the existing #4252 retry never fires because no
+  error is thrown to trigger it. The fix bounds each direct attempt's response-start wait to
+  `OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS` (30s default) via `directFetchWithBoundedResponseStart`, and retries
+  **once** on the same fresh no-keep-alive dispatcher from #4252 when that bound is hit.
+
+Both fixes assume the *socket* is the problem, not the upstream. `directFetchWithBoundedResponseStart`'s own
+implementation (`open-sse/utils/directResponseStartTimeout.ts`) is exactly two attempts: pooled, then fresh. When
+attempt 2 — a brand-new socket that cannot possibly be a zombie pooled connection — **also** times out at 30s,
+the retry logic has nothing left to try and the request fails with `DIRECT_RESPONSE_START_TIMEOUT_CODE`
+(surfaced as the 504 seen in the logs). A fresh socket succeeding to *connect* but the *server* never sending a
+response is exactly what "the upstream process is alive but stuck" looks like from OmniRoute's side — it can't
+distinguish "GPU is wedged mid-generation" from "stale pooled socket," because both present as "nothing came
+back in 30s." 46 consecutive both-attempts-failed cycles over 48 minutes is far outside what a transient stale-socket
+burst (the scenario #4252/#10214 were built for) would produce; it's consistent with a sustained upstream
+outage instead.
+
+**Not currently configured in this repo**: `grep`-ing `docker-compose.yml` and `.env.example` for
+`OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS` finds nothing — this deployment runs on the unmodified 30s default. (This
+is also, notably, a *fourth* data point alongside the two already-documented mechanisms and the `requestQueue.maxWaitMs`/
+`RATE_LIMIT_EXECUTION_TIMEOUT` timer mentioned in passing in `omniroute-account-semaphore-timeout.md` — OmniRoute
+has at least four independent timeout knobs guarding different stages of a request's life, three of them
+30-second-flavored by default, which is worth keeping in mind the next time an unfamiliar timeout string shows up.)
+
+## Why the upstream itself was likely unresponsive: ROCm/legacy-rocm-build#6630
+
+This session has **no SSH/shell access to the actual R9700 server** — there's no SSH config, and nothing in
+`scripts/` does remote exec, confirmed by inspecting `scripts/update.sh` and the absence of any `~/.ssh/config`
+entry for the box. So none of the following is confirmed against this specific incident's `dmesg`/`rocm-smi`/
+`docker logs` output — it's the closest primary-source match to the *symptom*, not a diagnosis of *this* outage.
+
+[ROCm/legacy-rocm-build#6630](https://github.com/ROCm/legacy-rocm-build/issues/6630) ("gfx1201 R9700 ROCm 7.14
+llama.cpp generation hang, MES queue failure and PSP reset -62; Vulkan passes") is an **open**, actively
+investigated issue (created 2026-08-19, most recent update 2026-08-28, no fix landed) that reproduces on **the
+exact same GPU this repo runs on** — Radeon AI PRO R9700, `gfx1201` — running **llama.cpp with `-fa on`** (this
+repo's `llama-server` also runs `--flash-attn on`), with a controlled Vulkan-vs-ROCm A/B: Vulkan completes
+normally, ROCm hangs during token generation. Direct quotes:
+
+> "During the ROCm generation stall: GPU busy reached 100%, memory busy remained 0%, VRAM use was only about 1.1
+> GB, **the container remained alive but made no output progress**."
+
+> "when MES stops responding, **the driver can stay unaware of it indefinitely** — the failure only surfaces when
+> something happens to send the next MES message... if a run is left alone after it stops making progress, the
+> kernel prints nothing at all, so a hang can look like a slow workload rather than a fault."
+
+> "the failure is probabilistic, not deterministic... a single passing run on this host does not indicate a
+> healthy configuration."
+
+The thread (12 comments as of this research pass, an AMD engineer `harkgill-amd` participating) has ruled out,
+one at a time, `uni_mes=0`, `mes_log_enable=1`, ROCm 6.4.4, ROCm 7.14, ROCm 10.0.0 stable, the latest TheRock
+nightly, and GFXOFF-disable — **no confirmed fix or workaround exists in the thread as of this research pass**.
+A related comment on the same issue (`chrisfranson`) reports the identical MES `REMOVE_QUEUE`/MODE1-reset
+signature from a **completely unrelated workload** (headless LibreOffice with OpenCL) on the same `gfx1201`
+silicon, reinforcing that this is a driver/firmware-level fault under general GPU load, not something specific
+to llama.cpp's request pattern.
+
+**This is a different bug from the one already mitigated in this repo.** `docs/research/rocm-gpu-pin-and-render-group.md`
+already documents and works around `ROCm/ROCm#5706` (clock/power pinned at boost whenever two concurrent HIP
+contexts share the GPU — fixed via `GPU_MAX_HW_QUEUES=1`, already set on both `llama-server` and `qwen-classifier`
+in `docker-compose.yml`). #5706's symptom is elevated power draw with the GPU still working; #6630's symptom is
+generation fully halting with `gpu_busy=100%`/`mem_busy=0%` and MES no longer responding at all — a real hang, not
+a clock-pin inefficiency. `GPU_MAX_HW_QUEUES=1` targets #5706's specific trigger (hardware-queue oversubscription
+across concurrent HIP processes) and has no evidence in #6630's thread of affecting that bug — #6630 reproduces
+in single-GPU, single-process benchmarks with no second HIP context involved at all, so the already-applied fix
+should not be assumed to help here.
+
+## What would actually resolve this vs. what wouldn't
+
+- **Raising `OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS`** — not recommended as a fix. It would make each failed attempt
+  take longer before giving up (worse latency during a real hang), without addressing why the GPU stopped
+  responding. It's the right lever only if future evidence shows genuinely-slow-but-working responses being
+  mistaken for hangs (the same shape as the already-fixed `REQUEST_TIMEOUT_MS` issue in
+  `omniroute-non-ping-sse-stream-timeout.md`) — this incident's 46-for-46 both-attempts-failed pattern over 48
+  minutes doesn't fit that shape.
+- **Concrete next step for whoever has server access when this recurs**: check `dmesg | grep -i amdgpu` and
+  `journalctl -k` on the R9700 host for `MES(...) failed to respond`, `GPU reset begin`, or `PSP resume failed`
+  lines matching #6630's signature, and `docker logs llama-server`/`docker logs qwen-classifier` to see whether
+  the process was alive-but-stuck (consistent with #6630) versus crashed/restarted (which would point elsewhere).
+  Capturing this during a live incident is the only way to move this from "best primary-source match" to
+  "confirmed root cause."
+- **Monitor [ROCm/legacy-rocm-build#6630](https://github.com/ROCm/legacy-rocm-build/issues/6630)** for a fix —
+  it's open and active (AMD engineer engaged as of 2026-08-26); no released ROCm version as of this research pass
+  is confirmed clean.
+- **The `-UD-Q8_K_XL.gguf` classifier variant in the pasted table but absent from version control** is worth a
+  direct look on the server (`cat .env` / `docker inspect qwen-classifier` for the actual `LLAMA_CLASSIFIER_MODEL_FILE`
+  in effect) — outside this research pass's reach without server access, flagged here so it isn't lost.
+
+## Recovery: how the hang actually clears (or doesn't) — addendum, 2026-09-15
+
+Follow-up question: what actually recovers the socket once this hits, given it's been observed to stay wedged
+for days at a time? Pulled the full comment thread on
+[ROCm/legacy-rocm-build#6630](https://github.com/ROCm/legacy-rocm-build/issues/6630) directly via the GitHub API
+(12 comments, `angelhalo` as primary reporter, `harkgill-amd` as the responding AMD engineer, plus one
+corroborating report from `chrisfranson` on unrelated hardware/workload) — the earlier research pass's source list
+cited this issue but hadn't read the full thread. Three things fall directly out of it:
+
+**There is no reliable in-band recovery.** The driver doesn't notice the hang on its own — direct quote:
+"when MES stops responding, the driver can stay unaware of it indefinitely — the failure only surfaces when
+something happens to send the next MES message." In a captured live hang, "every driver-managed ring is
+completely idle while the GPU reports 100% busy," `dmesg` has zero amdgpu lines, and no task is in D-state —
+so from the OS's perspective nothing is wrong; only sending the GPU another command (which killing/restarting
+the stuck process does) triggers the driver to discover the wedge and attempt its own MODE1 reset.
+
+**Once triggered, that reset itself is a coin flip across three documented outcomes**, not a guaranteed fix:
+1. **Clean recovery** — `GPU reset succeeded, trying to resume`, PSP resumes, the card comes back (VRAM is
+   wiped — "VRAM is lost due to GPU reset!" — so the container needs a real restart to reload the model, a plain
+   process respawn isn't enough even when the reset itself works).
+2. **Failed resume** — `PSP resume failed`, `GPU reset end with ret = -62` (the original report's own outcome) —
+   the reset attempt itself fails, leaving the GPU in a worse state than before.
+3. **Full kernel soft-lockup** — `chrisfranson`'s independent report (different workload — headless LibreOffice
+   OpenCL, different card — RX 9070 XT, same `gfx1201` silicon) hit outcome 2 or 3 twice out of three times:
+   "the whole system hard-locked (kernel soft lockup pegging a CPU at ~90-100% softirq, requiring a physical
+   power cycle)."
+
+`angelhalo` deliberately left one hang untouched rather than killing the process, to observe it without
+contaminating the state with a reset: "I recovered only with a subsequent cold power cycle" — no reset was ever
+triggered because nothing sent the GPU another message. Their standard test procedure between every single run
+in this thread is "a cold power cycle (AC removed, ≥30 s)," specifically **not** a warm/soft reboot — stated
+reason: "on this card a MODE1 reset takes the host down with it," meaning even the OS's own reboot path can't
+be trusted to come back cleanly once this GPU is in a bad state. This is the practical answer to "why does it
+stay stuck for days": nothing about the hang self-clears, `docker`'s `restart: unless-stopped` policy never
+fires because the container process is alive and never exits (confirmed: this repo's `llama-server` and
+`qwen-classifier` services have no `healthcheck` block at all — only `omniroute` itself does, a plain TCP
+connect check on its own dashboard port, which says nothing about whether `llama-server`/`qwen-classifier` are
+responding) — so a hang persists until a human notices the symptom (requests failing) and manually intervenes,
+and "days" is just however long that takes to notice on a homelab box, not a property of the hang itself.
+
+**No fix or reliable mitigation exists as of this reading (2026-08-28, the thread's latest comment).**
+`harkgill-amd` (AMD) could not reproduce locally and asked for a nightly-driver retest; `angelhalo` retested and
+it still failed. Every other variable tested still hangs: ROCm 6.4.4 through 10.0.0 stable, TheRock nightlies,
+`amdgpu.uni_mes=0`, `cwsr_enable=0`, `mes_log_enable=1`, GFXOFF disabled, two different physical R9700 cards, both
+llama.cpp and vLLM. The thread's own conclusion, as of the last comment: "a probabilistic lost-completion event"
+with no known trigger to avoid and no known driver/firmware combination that's clean.
+
+**Practical takeaway for this repo, given no upstream fix exists:**
+- A restart *might* recover it, *might* make it worse (failed PSP resume), and *might* take the whole host down
+  requiring a physical power cycle — there's no way to know in advance which outcome a given hang will produce.
+- Nothing currently watches for this automatically. Docker's `restart: unless-stopped` is the wrong tool (process
+  doesn't exit) — recovering automatically would need a `healthcheck` against `llama-server`'s own `/health`
+  endpoint (llama.cpp's built-in liveness endpoint) paired with something that acts on an `unhealthy` status,
+  since Docker itself doesn't restart on failed healthchecks without an external watcher (e.g. `willfarrell/autoheal`
+  or equivalent) — not evaluated here, flagged as a real gap, not a recommendation to implement blind: an
+  automated restart during a hang that's about to fail its PSP resume and lock the host could turn a
+  "requests are failing" incident into "the box needs a physical power cycle" automatically and unattended,
+  which is a real downside worth weighing against faster detection.
+- Given the reset outcome is unpredictable, the safest manual recovery when this is caught live is: restart the
+  affected container, then immediately check `dmesg | grep -i amdgpu` for `PSP resume failed` or a soft-lockup
+  signature before assuming it's fixed — if either appears, a full reboot (and per this thread's own testing
+  practice, possibly a genuine AC power cycle rather than a warm reboot) is the next step, not a second restart
+  attempt.
+
+## Caveats and open questions
+
+- **JSON export vs. pasted table are two different, non-overlapping captures.** The JSON file is a precise 6-hour
+  window with full per-request detail; the pasted table is a longer, dashboard-UI-copied range with less
+  structure and no verifiable overlap with the JSON file's timestamps. A fresh multi-day JSON export (same
+  `request-logs` endpoint used to produce the file analyzed here) would let a future pass check whether the
+  classifier's failures and the 27B model's outage are literally simultaneous (strong evidence for a shared
+  GPU-level cause) or independent recurrences of the same mechanism on separate schedules.
+- **Why the outage self-recovered after ~48 minutes with no observed restart is unexplained.** #6630's thread
+  describes hangs resolving via an explicit GPU reset (sometimes failing, requiring reboot) — not a case of a
+  hang clearing on its own after a fixed interval. Nothing in the available data (no server access) confirms
+  whether a restart happened that isn't visible from OmniRoute's logs, or whether this specific hang genuinely
+  self-cleared, which would be a data point *against* the #6630 hypothesis worth capturing next time.
+- **Live reproduction was not attempted.** The user suggested testing tool-calls against the classifier via the
+  Windows-side qwen-code CLI (`C:\Users\aerli\AppData\Local\qwen-code\bin\qwen.cmd`) to try to reproduce a
+  "Direct response did not start" failure live. Skipped for this pass: qwen-code requires an interactive/
+  already-authenticated session to drive meaningfully, and deliberately trying to reproduce a GPU hang against
+  the shared production classifier risked a genuine 60s+ stall on infrastructure other work depends on, for
+  uncertain diagnostic payoff given the strength of the log-based and source-based evidence already gathered.
+  Worth doing deliberately, with server access on hand to capture `rocm-smi`/`dmesg` simultaneously, rather than
+  as a quick check from this pass.
+
+## Sources
+
+- `omniroute-request-logs-6h-2026-09-15.json` — OmniRoute dashboard request-log export provided by the user
+  (2026-09-15, 339 entries, `12:49:52Z`-`18:44:54Z`)
+- User-pasted OmniRoute dashboard table (`qwen-classifier`/`qwen3-4b` failures, separate capture window)
+- [diegosouzapw/OmniRoute](https://github.com/diegosouzapw/OmniRoute) —
+  `open-sse/utils/directResponseStartTimeout.ts`, `open-sse/utils/proxyFetch.ts`, `open-sse/utils/proxyDispatcher.ts`,
+  `tests/unit/proxyfetch-direct-response-start-timeout-10214.test.ts`, `tests/unit/proxyfetch-retry-fresh-socket-4252.test.ts`,
+  `open-sse/handlers/chatCore/upstreamTimeouts.ts`
+- [ROCm/legacy-rocm-build#6630](https://github.com/ROCm/legacy-rocm-build/issues/6630) — open R9700/gfx1201
+  llama.cpp generation-hang issue, full comment thread (2026-08-19 through 2026-08-28)
+- [`docs/research/omniroute-account-semaphore-timeout.md`](./omniroute-account-semaphore-timeout.md) — the
+  first already-documented 30s OmniRoute timeout (account semaphore, 429, hardcoded)
+- [`docs/research/omniroute-non-ping-sse-stream-timeout.md`](./omniroute-non-ping-sse-stream-timeout.md) — the
+  second already-documented timeout (first-SSE-event deadline, `REQUEST_TIMEOUT_MS`-derived)
+- [`docs/research/rocm-gpu-pin-and-render-group.md`](./rocm-gpu-pin-and-render-group.md) — the already-mitigated,
+  *different* R9700/gfx1201 MES bug ([ROCm/ROCm#5706](https://github.com/ROCm/ROCm/issues/5706), clock-pin/power,
+  not a hang)
+- Local `docker-compose.yml`, `.env.example` (grepped directly, confirming `OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS`
+  is unset / on the 30s default, and that `GPU_MAX_HW_QUEUES=1` is already applied to both GPU services)
+- `git log --since=2026-09-14 --until=2026-09-16` (this repo, confirming zero commits during the outage window)
+
+## Confidence / uncertainty summary
+
+- **High confidence**: the exact mechanism and semantics of `OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS` /
+  `directFetchWithBoundedResponseStart` (read directly from OmniRoute's own source and its two regression test
+  files, which spell out the intent in comments referencing the originating issues); the JSON log's chronology,
+  single-connection scope, and the "two ~30s attempts, both failing" shape of every 504 (computed directly from
+  the log file); that this timeout is unconfigured in this repo (direct grep); that no commits landed in this
+  repo during the outage window (direct `git log`).
+- **Medium confidence**: that ROCm/legacy-rocm-build#6630 is the actual root cause of *this specific* outage.
+  The GPU model, driver-family symptom shape (`gpu_busy=100%`/`mem_busy=0%`, alive-but-stuck, sometimes
+  logged/sometimes silent), and `-fa on` usage all match closely, and it's an open/unresolved/actively-discussed
+  issue as of this research pass — but nothing from this incident's own `dmesg`/`rocm-smi` output was available
+  to confirm it directly (no SSH access), so this is the best primary-source match to the symptom, not a
+  confirmed diagnosis.
+- **Low confidence / open**: why the outage recovered on its own after ~48 minutes with no observed restart;
+  whether the classifier's pasted-table failures share the exact same triggering event as the 27B outage
+  analyzed here (same failure signature, but no verified time-overlap between the two data sources); the
+  provenance of the `-UD-Q8_K_XL.gguf` classifier variant seen in the pasted table but absent from version
+  control.
+
+## Live test, 2026-09-15: the classifier is measurably too slow at its own documented worst case — independent of any hang
+
+Before scoping a fix, tested the live `qwen-classifier` backend directly against realistic worst-case load,
+per the user's request to gather fresh evidence rather than design blind. Two attempts to reproduce this through
+qwen-code itself first surfaced an unrelated, separately-useful finding; the direct backend test below is what
+actually answered the question.
+
+### qwen-code's own headless mode never reaches the classifier
+
+Ran `qwen --approval-mode auto <prompt>` (positional/one-shot, non-interactive) from the Windows-side install
+(`C:\Users\aerli\AppData\Local\qwen-code\bin\qwen.cmd`, which has both `fastModel` and the `omniroute-search` MCP
+server already configured), asking it to run a shell `dir` and use the web-search MCP tool. Both attempts hit a
+wall before any classifier request was even sent:
+
+- The MCP tool call was refused outright: `Warning: Tool "mcp__omniroute-search__search" requires user approval
+  but cannot execute in non-interactive mode. ... use the -y flag (YOLO mode)`.
+- The shell tool: the model itself reported `run_shell_command` as "not registered" in this session and silently
+  substituted a read-only `glob` call instead — no approval prompt, no classifier call, no system warning printed
+  (unlike the MCP case), across two separate clean runs.
+
+**Conclusion: one-shot headless `qwen <prompt>` invocations don't exercise Auto Mode's classifier at all for
+approval-requiring tools** — they're declined or silently rerouted before the classifier ever gets a request.
+The classifier only fires in a genuinely interactive session, where it substitutes for the human's live approval
+decision. This wasn't previously documented anywhere in this repo and is worth keeping in mind: headless qwen-code
+testing is not a valid way to probe classifier behavior, live or otherwise. (Not investigated further: whether
+`qwen serve`/`--input-format stream-json` headless-agent modes behave differently — plausible, since they're
+built for exactly this kind of automation, but out of scope for this pass.)
+
+### Direct backend test: real classifier latency at realistic token counts
+
+Given headless qwen-code couldn't drive this, sent shaped classifier requests straight to
+`http://proxy-ai.home/v1/chat/completions` (model `qwen3-4b//models/Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf`,
+confirmed present in `GET /v1/models`) using a temporary scoped API key, mimicking the `{shouldBlock}`
+JSON-verdict shape and the `MAX_TRANSCRIPT_MESSAGES=40` / `MAX_HISTORICAL_ACTION_CHARS=4000` structure this
+repo's own `fast-model-choice.md` already read out of qwen-code's `classifier-transcript.ts` source. Five calls,
+in order:
+
+| # | Prompt tokens | `cached_tokens` | Wall time | Notes |
+|---|---|---|---|---|
+| 1 | 1,000 | 3 | 2.24s | Small prompt, genuinely fresh — healthy baseline. |
+| 2 | 51,234 | 51,233 | **64.21s** | First send of a large synthetic worst-case transcript (~40 highly self-similar 4K-char "historical action" blocks). Near-total cache hit reported, yet still the slowest call — see caveat below. |
+| 3 | 51,234 | 51,233 | 0.07s | **Identical repeat of #2.** Same `chatcmpl-...` id as #2 came back — this is OmniRoute short-circuiting an exact-duplicate request via a proxy-level response cache, not fresh inference. Confirms #2/#3's `cached_tokens` field is not a reliable proxy for wall-clock latency on its own. |
+| 4 | 1,000 | 3 | 0.03s | Identical repeat of #1 — same id, same response-cache short-circuit. |
+| 5 | 30,958 | 919 | **28.66s** | Fresh, non-repeated worst-case-shaped transcript (different random content, no internal self-similarity to trigger cache effects). Mostly-uncached (919/30,958) — this is the clean data point. |
+
+Call #5 is the one to trust: **~31K genuinely-fresh prompt tokens took 28.7 seconds** on the current
+`--n-gpu-layers 28` (of 36) / `--cache-type-k/v q4_0` / `--parallel 1` configuration, with the backend otherwise
+idle and healthy (no hang in progress). Extrapolating that rate to the repo's own documented worst case (a real
+15,116-token call observed live per `fast-model-choice.md`, and a theoretical ceiling around 40-50K tokens per
+`classifier-transcript.ts`'s limits) puts a genuine worst-case classifier call at **roughly 30-65 seconds of
+normal, non-hung processing time** — consistent with call #2's 64.21s, even though that call's own cache
+metadata is too muddied by internal prompt self-similarity to use as a second clean sample.
+
+**This directly overlaps both binding timeouts**: qwen-code's own client-side classifier stage timeout
+(`stage1Ms`/`stage2Ms`, `60000` each in the Windows-side `settings.json` observed this session, `30000`/`60000`
+in the WSL-side one) and `OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS`'s 30-second-per-attempt window documented above.
+A worst-case classifier call landing in the 30-65s range will **routinely** trip one or both of these timeouts
+on its own, with the backend never having hung at all — the exact same 499/504/"Request aborted" shape as the
+GPU-hang hypothesis produces, but from an entirely mundane, deterministic cause: **the classifier's current
+configuration is simply too slow for the request sizes qwen-code's own classifier-transcript design allows.**
+
+### What this changes
+
+This doesn't rule out ROCm/legacy-rocm-build#6630 — the 27B model's 48-minute total outage (zero successes, not
+just slow ones) doesn't fit a "just slow" explanation, and remains best matched by a real GPU hang. But it does
+mean **the classifier's own recurring failures (the pasted-table evidence) very plausibly have a second,
+independent, non-probabilistic cause that a healthcheck/restart-on-hang design wouldn't fix at all** — restarting
+a classifier that's merely slow-but-working at worst-case load just interrupts a call that would have succeeded,
+and would fire repeatedly under normal peak usage, not just during a rare hang. Any fix that only targets "detect
+and recover from an unresponsive GPU" leaves this second failure mode untouched. Two independent levers worth
+weighing before finalizing a scope: raising the classifier's own timeouts to match its real worst-case latency
+(cheap, immediate, but does nothing for actual hangs), and/or speeding up the classifier itself (full GPU offload
+if VRAM allows, a faster quant, or capping the transcript size client-side) to bring worst-case latency back
+under the existing timeouts.
+
+**Not investigated in this pass**: whether call #2's 64.21s (vs. call #5's extrapolated ~45-48s at a similar
+token count) reflects genuine non-linear slowdown at the very largest context sizes, real concurrent contention
+from other production traffic sharing the same `--parallel 1` slot during the test, or is just noise from a
+single sample each — worth a few more clean, uniquely-content, worst-case-sized calls at different times of day
+before treating either number as precise.

docs/research/omniroute-memory-tool-injection-breaks-classifier.md

@@ -0,0 +1,216 @@
+# OmniRoute's builtin memory tools silently hijack qwen-code's classifier tool-call, not a model or GPU problem
+
+**Date:** 2026-09-15
+
+**Verdict:** The `"Classifier stage 1 unavailable"` / `"Auto Mode couldn't classify this action"` failures are **not**
+a GPU hang, not a timeout, and not a Qwen3-4B quality problem. Confirmed directly from a live debug log: the fast
+model (`qwen3-4b//models/Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf`) *is* being used (`stage=fast` in every classifier
+log line), and it responds well within its timeout (18.7s against a 30-60s budget). The failure is
+`"Error: Invalid side query response: params must have required property 'shouldBlock'"` — a **schema-validation
+failure on an in-time response**. Root cause, confirmed directly against OmniRoute's own source
+(`open-sse/handlers/chatCore/memorySkillsInjection.ts`): **OmniRoute silently appends its own builtin memory tools
+(`memory_save`/`update`/`search`/`delete`) to every non-streaming chat completion's `tools` array**, whenever
+memory is enabled for the calling key — regardless of what tools the caller declared. qwen-code's classifier forces
+`tool_choice: ANY` (call *some* tool, not a specific one) so it can get a structured `respond_in_schema` JSON
+response. With OmniRoute's extra tools spliced in, the small model sometimes picks the injected `memory_save` tool
+instead — and since qwen-code's client only extracts the classifier's answer from a `respond_in_schema` function
+call (not from a stray `memory_save` call, even if the answer also happens to be present as plain text), the
+result validated against `STAGE1_SCHEMA` is empty, producing exactly the observed error.
+
+## The debug-log evidence
+
+Captured directly from a `-d` (debug) qwen-code run, `C:\Users\aerli\.qwen\debug\886d00eb-...txt`:
+
+```
+21:09:56 [DEBUG] [CLASSIFIER] ALLOW stage=fast tool=mcp__omniroute-search__search durationMs=15412
+21:10:28 [WARN]  [CLASSIFIER] failUnavailable stage=fast durationMs=18727 reason="Classifier stage 1 unavailable" cause="Error: Invalid side query response: params must have required property 'shouldBlock'"
+```
+
+Both lines are tagged `stage=fast` — qwen-code's own internal label confirming the classifier used the configured
+fast model both times, settling a live question this session raised about whether the classifier was silently
+falling back to the main 27B model. It wasn't. The second call's 18.7s duration is comfortably inside this
+deployment's configured `stage1Ms` (30000-60000ms depending on install) — this was not a timeout.
+
+## Reproduced directly
+
+Sent the exact same request shape qwen-code's `classifyAction()`/`generateJson()` builds — a forced tool call to a
+synthetic `respond_in_schema` function with `STAGE1_SCHEMA` (`{required: ["shouldBlock"], properties: {shouldBlock:
+{type: "boolean"}}}`) as its only declared tool, `tool_choice: "required"` — directly against
+`POST /v1/chat/completions` on this deployment (model `qwen3-4b//models/Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf`).
+The very first response:
+
+```json
+{
+  "message": {
+    "content": "{\"shouldBlock\": true}",
+    "tool_calls": [{
+      "type": "function",
+      "function": {
+        "name": "memory_save",
+        "arguments": "{\"key\": \"tool_call_review\", \"content\": \"web_fetch call to https://www.swfte.com/ai/leaderboard was blocked due to potential data exfiltration or unauthorized external access.\", \"type\": \"factual\"}"
+      }
+    }]
+  },
+  "tool_results": [{ "output": "{\"success\":true,\"memory\":{\"id\":\"84389721-...\", ...}, \"message\":\"Memory saved successfully\"}" }]
+}
+```
+
+`memory_save` was never declared in the request's `tools` array — only `respond_in_schema` was. OmniRoute added it
+server-side, the model called it instead of (or alongside) the intended tool, and OmniRoute **auto-executed it**
+(`tool_results` shows a real memory record was created, with a UUID and timestamp — this isn't inert). The correct
+`{"shouldBlock": true}` answer is sitting right there in `content`, but qwen-code's `generateJson()` only ever
+looks for it inside a `respond_in_schema` function call (`functionCalls.find(call => call.name ===
+"respond_in_schema")`); when that's absent it falls through to a raw-text JSON parse of `getResponseText(result)`
+— and OpenAI-client conventions generally don't populate a response's plain-text field when `finish_reason` is
+`tool_calls` rather than `stop`, so that fallback likely returns nothing, yielding `{}` against a schema that
+requires `shouldBlock`. This is a single confirmed repro, not a guaranteed-every-time reproduction — the injected
+tool doesn't win every time (an immediate identical follow-up request returned an OmniRoute-cached copy of the same
+response, not a fresh sample — see the cache caveat in
+[`omniroute-direct-response-timeout-outage-2026-09-15.md`](./omniroute-direct-response-timeout-outage-2026-09-15.md)),
+but it reproduces the *exact* failure shape from the live debug log on the first genuine attempt.
+
+## Root cause, confirmed in OmniRoute's own source
+
+[diegosouzapw/OmniRoute](https://github.com/diegosouzapw/OmniRoute) —
+`open-sse/handlers/chatCore/memorySkillsInjection.ts`:
+
+```ts
+if (memoryOwnerId && memorySettings?.enabled && body.stream !== true) {
+  // Server-side builtin memory tools (memory_save/update/search/delete) are
+  // executed by the gateway's tool-call interception, which runs only on the
+  // non-stream path. Stream clients (opencode etc.) execute tools client-side,
+  // so for them these tools would be announced but never executed; they should
+  // use the MCP memory tools (omniroute_memory_*) instead.
+  const existingTools = Array.isArray(body.tools) ? body.tools : [];
+  ...
+  const memoryTools = buildMemoryToolsForProvider(...).filter(tool => !existingToolNames.has(name));
+  if (memoryTools.length > 0) {
+    body = { ...body, tools: [...existingTools, ...memoryTools] };
+  }
+}
+```
+
+This runs unconditionally for any non-streaming request from a key with memory enabled — there is no exemption
+for a caller that already set `tool_choice` to force a *specific* tool. qwen-code's classifier is exactly this
+case: a single-purpose, forced-`ANY`, non-streaming tool call, which is precisely the shape this injection logic
+was not written to avoid interfering with.
+
+`src/lib/memory/settings.ts` confirms `enabled: false` is the *default* — memory is off by default in a fresh
+OmniRoute install specifically because of injected-context cost, per its own comment:
+
+> "Off by default: enabling memory injects up to `maxTokens` (~2k) of retrieved context into every chat request,
+> which is billed — a surprising cost for new installs... Opt in explicitly via Settings → Memory... Per-request
+> opt-out is also available via the `x-omniroute-no-memory` header."
+
+This deployment has memory enabled (confirmed live by the reproduction above), which is presumably a deliberate
+choice for other workflows (chat memory across sessions) — but it has an undocumented-to-this-repo side effect on
+any caller using forced-tool-call classification.
+
+## What would fix this
+
+Two per-target exclusions were checked live against this deployment and confirmed **not to exist**:
+
+- **Per-API-key memory override**: `GET /api/keys` was fetched directly (the temporary key handed to this session
+  turned out to carry admin access, well beyond the plain `/v1` workload scope its name implied). Every key's full
+  field list was inspected — `noLog`, `scopes`, `allowedModels`, `rateLimits`, `disableNonPublicModels`, etc. — with
+  no memory-related field anywhere.
+- **Per-model/connection override**: `GET /api/providers/<id>` for the classifier's own connection
+  (`qwen3-4b`, id `b78ceb4c-52f8-47ae-b245-483baa6e3fc2`) was fetched directly. `providerSpecificData` (`prefix`,
+  `apiType`, `baseUrl`, `nodeName`, `timeoutMs`, `apiKeyHealth`) has no memory field either — consistent with the
+  source: `memoryOwnerId` is resolved purely from the *calling key* (`resolveMemoryOwnerId(apiKeyInfo)`), before
+  OmniRoute has even picked a provider, so it can't know or care that this particular request targets the
+  classifier model specifically.
+
+**The fix that was actually available and is now applied**: `x-omniroute-no-memory`, OmniRoute's own per-*request*
+opt-out (not per-key or per-model), confirmed end-to-end and traced through both sides:
+
+- OmniRoute's handling, confirmed directly in `open-sse/handlers/chatCore.ts` and its own test suite
+  (`tests/unit/no-memory-header.test.ts`): `memoryOwnerId = isNoMemoryRequested(headers) ? null : resolveMemoryOwnerId(...)`
+  — a null owner id short-circuits *both* branches in `injectMemoryAndSkills` (context injection and tool
+  injection). The test suite gives the exact accepted values: `"true"`, `"1"`, `"yes"` (case-insensitive on both
+  the header name and value); `"false"`/`"0"`/`"no"`/empty do not trigger it.
+- qwen-code's support for sending it, confirmed against the installed bundle, *not* just the docs: `modelProviders.
+  openai[].generationConfig.customHeaders` (documented at
+  [model-providers](https://qwenlm.github.io/qwen-code-docs/en/users/configuration/model-providers/)) flows into
+  `DefaultOpenAICompatibleProvider.buildClient()` (`chunk-CXTPVBFA.js`), which passes it straight into the
+  underlying `OpenAI` SDK client as `defaultHeaders` — applied to every request made through that one model entry,
+  and *only* that entry (confirmed this is the generic OpenAI-compatible-chat client, the same one the classifier's
+  `apiType: "chat"` connection uses — not Anthropic- or Responses-API-specific plumbing).
+
+**Applied**, 2026-09-15: added to the `qwen3-4b-classifier` entry in the Windows-side `~/.qwen/settings.json`
+(`C:\Users\aerli\.qwen\settings.json`), inside its `generationConfig`, alongside the existing `contextWindowSize`
+and `extra_body`:
+
+```json
+"customHeaders": { "x-omniroute-no-memory": "true" }
+```
+
+Scoped to this one model entry only — the main `qwen3.8-27b-local` connection's `generationConfig` is untouched,
+so its own memory-context behavior (if any is relied on elsewhere) is unaffected. qwen-code's own
+`[MODEL_PROVIDERS_HOT_RELOAD]` settings watcher (confirmed present in this session's debug log) should pick this
+up on the already-running session without a restart. **Not yet verified live** — the next classifier failure (or a
+deliberate repro, per the "Reproduced directly" section above) should confirm no `memory_save`-shaped tool call
+appears in the response once this is in effect.
+
+- **Remaining fallback, if the header approach doesn't hold up**: disable memory globally for this deployment
+  (`PATCH /api/settings/memory`, `enabled: false`, or Settings → Memory in the dashboard) — blunt, but confirmed to
+  work by definition since `enabled: false` is every fresh install's default.
+- **Also worth doing regardless**: file this upstream with OmniRoute. Their own code already special-cases one
+  caller type (streaming clients) right next to this injection logic; a similar exemption for a caller that already
+  set `tool_choice` to force one specific tool would be a clean fix on their end that doesn't depend on every
+  client remembering to send an opt-out header.
+- **Not a fix, and not the problem**: nothing on the classifier-model or llama.cpp side. Qwen3-4B-Instruct-2507
+  correctly produced the right answer (`{"shouldBlock": true}`) in the one reproduction captured here — the model
+  was never at fault.
+
+## Scope note
+
+This session's earlier hypothesis that the 27B model's 48-minute total outage
+([`omniroute-direct-response-timeout-outage-2026-09-15.md`](./omniroute-direct-response-timeout-outage-2026-09-15.md))
+was caused by a ROCm/gfx1201 GPU hang is set aside here per explicit direction, not retracted — that was a
+different incident (zero successes for 48 straight minutes, a shape this memory-injection bug doesn't produce) and
+this finding doesn't bear on it either way.
+
+## Sources
+
+- Live debug log, `C:\Users\aerli\.qwen\debug\886d00eb-5b2b-4d84-b1ef-60909f75eec2.txt` (this session, 2026-09-15)
+- Direct reproduction against this deployment's `POST /v1/chat/completions` (this session, 2026-09-15)
+- Live `GET /api/keys`, `GET /api/providers`, `GET /api/providers/b78ceb4c-52f8-47ae-b245-483baa6e3fc2`,
+  `GET /api/settings/memory` against this deployment's OmniRoute instance (this session, 2026-09-15) — confirmed no
+  per-key or per-connection memory field exists in either schema
+- [diegosouzapw/OmniRoute](https://github.com/diegosouzapw/OmniRoute) —
+  `open-sse/handlers/chatCore/memorySkillsInjection.ts`, `open-sse/handlers/chatCore.ts` (the
+  `isNoMemoryRequested`/`resolveMemoryOwnerId` branch), `src/lib/memory/settings.ts`, `src/lib/memory/injection.ts`,
+  `open-sse/mcp-server/tools/memoryTools.ts`, `tests/unit/no-memory-header.test.ts` (exact accepted header
+  name/value set)
+- [Qwen Code docs — Model Providers](https://qwenlm.github.io/qwen-code-docs/en/users/configuration/model-providers/)
+  (`customHeaders` field, documented under `generationConfig`)
+- Installed qwen-code bundle — `chunk-N7VWZDWW.js`, `chunk-HBU7EKY4.js` (`classifyAction`, `runSideQuery`,
+  `resolveDefaultModel`, `generateJson`, `resolveFastModelSelector`, `getFastModel`) and `chunk-CXTPVBFA.js`
+  (`DefaultOpenAICompatibleProvider.buildHeaders()`/`buildClient()`, confirming `customHeaders` reaches the actual
+  OpenAI SDK client as `defaultHeaders` for the plain chat-completions path the classifier uses) — all read
+  directly from the bundled (unminified variable names) source, not inferred from docs alone
+- Applied fix: `C:\Users\aerli\.qwen\settings.json`, `qwen3-4b-classifier` entry's `generationConfig.customHeaders`
+  (this session, 2026-09-15)
+
+## Confidence / uncertainty summary
+
+- **High confidence**: the fast model is genuinely used for classification (`stage=fast` in qwen-code's own debug
+  log, both on success and failure); the failure is a schema-validation error on an in-time response, not a
+  timeout (18.7s duration, explicit error text); OmniRoute's `memorySkillsInjection.ts` unconditionally injects
+  builtin memory tools into non-streaming completions for any memory-enabled key, with no exemption for
+  forced-single-tool callers (read directly from source); no per-key or per-model/connection memory override
+  exists in this OmniRoute version (confirmed by reading the complete live schema of both, not by absence of
+  documentation); `x-omniroute-no-memory: true` is a real, working per-request opt-out on OmniRoute's side (its
+  own test suite) and is reachable from qwen-code via `modelProviders.openai[].generationConfig.customHeaders`,
+  traced to the exact HTTP client the classifier's connection type uses (not inferred from docs alone — confirmed
+  against the bundled source's actual header-merging code).
+- **Medium confidence**: that this exact tool-injection mechanism explains the *specific* production failures seen
+  earlier in this session's testing (the reproduction matches the failure shape and the source confirms the
+  mechanism exists and applies to this call pattern, but the live debug-log failure itself wasn't captured
+  mid-flight with response inspection — only its aftermath, the error message).
+- **Low confidence / not verified**: the exact conditions under which the model picks the injected tool over the
+  intended one (one clean reproduction on the first attempt, not a characterized hit rate — the failure may not be
+  deterministic, so the `customHeaders` fix should still be watched rather than assumed to have fully resolved it
+  on the strength of this write-up alone); whether the applied `customHeaders` fix has been confirmed live yet
+  (not as of this writing — see "Applied" above).

docs/research/qwen3.8-27b-decode-speed.md

@@ -0,0 +1,226 @@
+# Research: decode tokens/s on the R9700 (llama-server, Qwen3.8-27B + DFlash2)
+
+**Date:** 2026-10-20
+
+**Question:** Why is decode at ~52k context about plain-decode speed (26-28 t/s) when an older, shorter-context log averaged 45.9 t/s, and what can be changed to raise decode (and prefill where relevant)? Evidence: `llmserver.log` (re-analysed per request) plus llama.cpp source.
+
+**Scope note:** research only. Nothing in compose or `.env.example` was changed and nothing was deployed. This note builds on [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md) (cache/prefill ranking, DDR tier, five outside configs) and [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) (quant choice, VRAM budget). It does not repeat them, and it **corrects one premise of the first** (see finding 1).
+
+**Labels.** **[V]** verified from source (URL pinned to a commit, or a repo file:line). **[L]** computed from `llmserver.log` by me. **[R]** reported by a third party (a page or thread, not re-run). **[I]** inferred, not measured. WebFetch summaries of web pages were produced by a small model, so every number taken from a page is **[R]** even when the page is first-party.
+
+**Source pin.** llama.cpp `master` at `839a572a4cdecdb80d211297b8e7d7323a6e99d2` (fetched 2026-10-20). Links below point at that commit. The running image (`server-rocm`) may be older than this commit. Its build string is not in the logs I have, so every "[V]" is "true on that commit".
+
+## TL;DR
+
+| # | Finding | Label |
+|---|---|---|
+| 1 | The draft length has been **5, not 7**, in the old log too. All 306 `draft acceptance` summaries have `generated` divisible by 5 and `mean len = 1 + accepted / (generated / 5)`. So `n_max=5` in the new startup log is the same setting, not a new clamp. The earlier note's "n_max is 7" came from compose and was wrong | [L] + [V] |
+| 2 | master clamps `n_max` to `block_size - 1` (= 7 for block_size 8). 5 is below that, so no clamp fired. 5 must have been **requested**: most likely `LLAMA_SPEC_N_MAX=5` in the server's `.env` (the repo `.env.example` default differs, and `.env` is not in the repo) | [V] + [I] |
+| 3 | Decode speed tracks acceptance almost linearly (r = 0.97 over 256 requests) and acceptance falls with depth: 0.75 at <8k, 0.57 at 8-20k, 0.42 at 20-40k, 0.40 at 40-60k, while decode falls 64.6 to 31.9 t/s. The 45.9 t/s average was dominated by short-context requests | [L] |
+| 4 | Bandwidth ceiling for plain decode: 17.6 GB / 640 GB/s = 27.5 ms = **36.4 t/s** (33.0 t/s at 52k depth with q8_0 KV). At the 79-81 % efficiency best seen for dense models on this card that is 26-27 t/s, which is exactly the measured "plain" rate | [I] on [R] inputs |
+| 5 | A spec step costs 74-94 ms in the old log, against a ~28-30 ms weight-streaming floor. About 45 ms per step is fixed overhead (drafter, 6-wide verify, sync, sampling), so speculation only pays when acceptance is high | [L] + [I] |
+| 6 | DFlash2 draws its draft greedily and gives the server no draft distribution, so at temp > 0 a draft token is accepted only if the target's own sampled token equals it. Sampler settings on the client therefore change acceptance directly | [V] |
+| 7 | `GPU_MAX_HW_QUEUES=1` is free for default llama.cpp: multi-stream graph execution is opt-in (`GGML_CUDA_GRAPH_OPT=1`) | [V] |
+
+Top 3 experiments (details in "Experiment plan"): **A** sweep `--spec-draft-n-max` 3, 4, 5, 7 and add `--spec-draft-p-min`; **B** measure acceptance versus client sampler settings with a fixed prompt (temp, top-p/top-k, penalties); **C** decompose the 74-94 ms step with `llama-bench` (pp6 at depth vs tg1) and `rocm-smi` clocks, before touching KV type, slots or quant.
+
+## 1. DFlash2: why acceptance and speedup vary
+
+### 1.1 The n_max 5 vs 7 discrepancy
+
+- The init line prints `n_max=%d` from `this->params.n_max` **before** the clamp: [`common/speculative.cpp#L1086`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1086). The clamp is [`#L1090-L1099`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1090): `n_draft_max = (is_dspark && sample_from_anchor) ? block_size : block_size - 1`, with the warning `requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d`. **[V]**
+- For this drafter (`block_size=8`, DFlash2 is not DSpark) the limit is 7. So a requested 7 passes unclamped and a requested 5 prints 5. The init line therefore shows the **requested** value; no clamp explains 5. **[V]** (on that commit)
+- `n_extract=5` in the log is `target_layer_ids_n`, the number of target layers whose features feed the drafter ([`#L1040-L1044`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1040)). It is not used in the clamp, so the 5 = 5 match is a coincidence on this commit. **[V]** An older build could clamp differently; I could not check history for this. **[I]**
+- Log proof that 5 is the real draft length. The server prints `mean len = 1 + accepted / verif_steps` ([`tools/server/server-context.cpp#L704-L725`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L704)). With `verif_steps = generated / 5` this reproduces the printed mean len in all 306 summaries; with 7 it matches none. Examples: `llmserver.log:3` (47 accepted / 105 generated, 21 steps, 47/21 + 1 = 3.24), `:36` (64/170, 34 steps, 2.88), `:114` (828/2050, 410 steps, 3.02). **[L]**
+- To confirm on the server: `grep -n SPEC_N_MAX .env` and `docker logs llama-server | grep -E "n_max|exceeds the trained"`. **[I]**, not run.
+
+### 1.2 What the drafter does per step
+
+- The drafter input is `[id_last, <mask> * n_max]` and one forward pass denoises the whole block. DFlash2 then greedily walks a candidate lattice (`selector_top_k` candidates per position, argmax per step): [`#L1296-L1310`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1296) and [`#L1336-L1366`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1336). The block size in the drafter batch is `n_max + 1`, so a smaller `n_max` shrinks the drafter batch too. **[V]**
+- `--spec-draft-p-min` on DFlash2: stop extending the draft at the first position where softmax(selector scores) at the argmax is below `p_min` ([`#L1349-L1358`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1349)). It is a per-position stop, default 0.00 ([`common/common.h#L333`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L333), [`common/arg.cpp#L4248`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4248)). The git log shows DFlash2-specific p_min arrived with the DFlash2 PR ([#27342](https://github.com/ggml-org/llama.cpp/pull/27342), commit titles "Add p_min in DFlash2" per the PR page **[R]**). **[V]** for the code. The `generated` multiples of 5 in the log confirm p_min was 0 in the old run. **[L]**
+- `n_min` (`--spec-draft-n-min`, default 0): a draft shorter than `n_min` is discarded ([`#L1362-L1364`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1362)). **[V]**
+- `sample_from_anchor=true` only changes the layout for DSpark drafters; for DFlash2 the block is `n_max + 1` ([`#L1298`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1298)). **[V]**
+- `--spec-draft-sampling probabilistic` ([`arg.cpp#L4264`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4264)) is documented as sampling the draft and verifying by rejection. The DFlash2 branch never fills the draft distribution (`result_q`), and the server then falls back to plain sample-and-match: `use_rejection = use_spec_rejection() && !spec_draft_q.empty()` ([`server-context.cpp#L4537`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L4537)). So that flag does nothing for DFlash2. **[V]** (read of the code path; not run).
+
+### 1.3 Why acceptance varies
+
+1. **Sampling.** With the DFlash2 drafter, a draft token is accepted iff the target's sampled token equals it ([`common/sampling.cpp#L690-L720`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/sampling.cpp#L690), `draft[i] != id` breaks). At temp 0 the target token is its argmax, the best case for matching a greedy draft. At temp > 0 it falls with flatness of the target distribution and with anything that reshapes it (top-p/top-k, repetition or presence penalties). **[V]** for the mechanism. The size of the effect for this workload is unmeasured. **[I]**
+   - Contradiction to resolve: the earlier note cites a third-party claim that greedy sampling "collapses" DFlash2 acceptance. The source reading says the opposite for the server path. Treat that claim as unverified until experiment B settles it.
+   - Server-side the temp > 0 test is `task->params.sampling.temp > 0` ([`server-context.cpp#L508`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L508)). The drafter itself ignores temperature. **[V]**
+2. **Context depth.** In the old log, acceptance halves from <8k to >40k tokens (0.75 to 0.40, table below). A PR commenter also reported code prompts decaying faster with context length than prose ([PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) comments) **[R]**. The model card gives no context length for the drafter. **[R]**
+3. **Content type.** Not separable in this log (no prompt/response classes). The authors' acceptance lengths (mean tokens per verify step, 7 drafted, block size 8, temp 1.0 / top-p 0.95 / top-k 20, SGLang on an H200) are GSM8K 5.46, MATH-500 5.28, HumanEval 4.39, MBPP 4.79 ([z-lab/Qwen3.8-27B-DFlash2](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2), via a search summary) **[R]**. Your mean len of 2.9-4.9 (max possible 6 at n_max 5) is below that, partly because those are short prompts. **[I]**
+4. **Concurrency.** Issue [#27117](https://github.com/ggml-org/llama.cpp/issues/27117) (gfx1151, closed "not planned"): acceptance collapses as the number of concurrently drafting sequences grows (0.51-0.66 at 4 active, 0.06-0.49 at 16), and `--spec-draft-n-max 1` recovered it. **[R]** Your two slots are far below that regime, but the bimodal per-slot acceptance in that issue is worth watching if both slots decode at once. **[I]**
+
+### 1.4 Authors' recommended values
+
+- Model card quick start: `--spec-type draft-dflash --spec-draft-n-max 7` ([z-lab/Qwen3.8-27B-DFlash2-GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF)) **[R]**. No block size, temperature or p_min advice is given there. The card says the output is lossless (greedy identical, sampling distribution preserved).
+- A PR commenter on an RTX 3090 used `--spec-draft-n-max 4` and reported that 7 was slower: DFlash2 at n=4 averaged 62.21 t/s at 0.6153 acceptance, MTP 57.79 t/s at 0.6317 ([PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) comments) **[R]**.
+- Drafter quant does not matter for acceptance: acceptance length 5.28 (BF16), 5.13 (Q8_0), 5.39 (Q4_K_M) on 8 GSM8K prompts; the card itself implies Q4_K_M above BF16 is noise. Q4_K_M is ~1.1 GB, so it also costs the least bandwidth. **[R]**
+
+### 1.5 What the old log shows (re-analysis)
+
+Per-request join of `stop processing: n_tokens` (final depth), `eval time` t/s and `draft acceptance`, requests with >= 200 generated tokens, 256 requests (script: ad-hoc Python over `llmserver.log`, not committed). Draft length 5, so tokens per verify step is 1 + 5 x acceptance.
+
+| Final depth | n | Decode t/s (mean) | Acceptance (mean) | Tokens/step | Step time (tokens/step / t/s) |
+|---|---|---|---|---|---|
+| < 8k | 125 | 64.6 | 0.75 | 4.75 | ~74 ms |
+| 8-20k | 28 | 44.2 | 0.57 | 3.85 | ~87 ms |
+| 20-40k | 17 | 34.6 | 0.42 | 3.10 | ~90 ms |
+| 40-60k | 86 | 31.9 | 0.40 | 3.00 | ~94 ms |
+
+- Correlation of per-request decode t/s with acceptance: r = 0.97, slope ~90 t/s per 1.0 acceptance. **[L]**
+- Step time rises only ~20 ms from 0 to 50k depth, while tokens per step drops 4.75 to 3.0. So about **80 % of the decode loss with depth is acceptance, not attention cost**. **[L]** (derived, mean-of-ratios caveat).
+- The new run (26-28 t/s at ~52k, tg_3s 20-33) is at or just below the 40-60k bucket (31.9), consistent with the same acceptance regime. **[L]**
+
+## 2. Decode speed versus context depth
+
+### 2.1 What the attention costs here
+
+- Only 16 of 64 layers keep a KV cache ([`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) "Model identity"). KV bytes per token: 16 layers x 2 x 4 heads x 256 x element size = 65,536 B (f16), 34,816 B (q8_0, 1.0625 B/element), 18,432 B (q4_0, 0.5625 B/element). At 52k tokens: 3.4 GB (f16), 1.81 GB (q8_0), 0.96 GB (q4_0). Against 17.6 GB of weights read per token, q8_0 KV adds ~10 % bytes at 52k, f16 ~19 %, q4_0 ~5 %. **[I]** (arithmetic from the quant note's architecture facts.)
+- So KV precision can move plain decode at 52k by at most a few percent. Going to q4_0 would buy ~1 GB less read (~5 %) at an unmeasured quality cost. Do not expect it to fix the 2x gap. **[I]**
+
+### 2.2 Which flash-attention kernel runs on gfx1201
+
+From [`ggml/src/ggml-cuda/fattn.cu#L540-L722`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L540) (`ggml_cuda_get_best_fattn_kernel`), applied to head_dim 256, 24 query heads, 4 KV heads (gqa_ratio 6, effective ratio 2), K/V q8_0:
+
+| Query batch (tokens) | Kernel on gfx1201 | Why (source) |
+|---|---|---|
+| 1 (plain decode) | VEC | WMMA needs `Q->ne[1] * gqa_ratio_eff > 16` for head_dim > 128 ([`#L698-L702`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L698)), so 1 x 2 fails. For quantized K/V the vector kernel is chosen when `Q->ne[1] <= 2` ([`#L714-L720`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L714)). Reads q8_0 directly. |
+| 2 | VEC (quantized) | same rule |
+| 3 to 8 (verify batch of a spec step) | **TILE** | 8 x 2 = 16 is not > 16, so no WMMA; quantized KV is not eligible for VEC above 2 tokens. |
+
+- TILE and MMA kernels "need f16 K/V" ([`#L735-L740`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L735)): with a quantized cache the K and V views are converted to f16 scratch before the kernel runs. So a verify step at 52k converts ~100M q8_0 elements per layer (K plus V) to f16 on 16 layers. Rough extra traffic: read 1.8 GB plus write 3.4 GB plus read 3.4 GB per step, ~8.6 GB, ~13 ms at 640 GB/s. **[I]** (the conversion and kernel choice are [V]; the traffic estimate and its cost are inferred, not profiled). That is consistent with the ~20 ms step-time rise from 0 to 50k in 1.5. With f16 KV the conversion disappears (TILE reads f16 directly) but the cache is 1.9x larger, so the net is not obvious. **[I]**
+- No kernel-level bug report for gfx1201 + quantized KV turned up in the sources I read. The only RDNA4 data point is [R]: on RADV (Vulkan) q8_0 vs q4_0 KV was within +-0.5 % ([discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043)). That is a different backend and says nothing about the HIP path.
+
+### 2.3 `--parallel 2` and `--kv-unified`
+
+- With `--kv-unified` there is one KV stream. `get_n_kv` pads `cells.used_max_p1()`, the highest occupied cell across **all** sequences ([`src/llama-kv-cache.cpp#L1282-L1292`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-kv-cache.cpp#L1282)). The flash-attention kernels only skip fully masked tiles at the **end** of the range (`flash_attn_mask_to_KV_max`, [`fattn-common.cuh#L671-L723`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn-common.cuh#L671)). So when the other slot holds a long prompt in higher cells, slot 0's decode may read and mask-skip past tens of thousands of foreign cells. **[V]** for both pieces, **[I]** for the combined cost (not profiled; which cells each slot occupies is not logged).
+- This matches the pattern in the older log (requests alternate slot 0 / slot 1, each ~40-50k tokens, [throughput note](qwen3.8-27b-throughput-tuning.md) "Pattern at `:380`-`:470`"), but no measurement separates it. **Experiment D** below tests it.
+- `--kv-unified-per-slot N` exists ([`arg.cpp#L1652`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L1652)) and `--no-kv-unified` gives one stream per slot ([`arg.cpp#L1727-L1734`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L1727)). Non-unified mode splits `--ctx-size` evenly, so one slot could no longer use more than half the context. **[V]**
+
+### 2.4 Checkpoints and recurrent state during decode
+
+- The target is `qwen35` architecture, which `llm_arch_supports_rs_rollback` lists ([`src/llama-arch.cpp#L1256-L1262`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-arch.cpp#L1256)). I am assuming Qwen3.8 loads as that arch; the name was not verified. **[I]**
+- For DFlash/DSpark the server requests `n_rs_seq = draft.n_max` recurrent-state snapshots per sequence ([`common/common.h#L401-L407`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L401); [`src/llama-context.cpp#L191-L195`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-context.cpp#L191)). The server creates a **speculative checkpoint** only if `draft.size() > llama_n_rs_seq(ctx_tgt)` ([`server-context.cpp#L3541-L3558`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L3541)), and restores one only if `n_rollback > n_rs_seq` ([`#L4559`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L4559)). With `n_rs_seq == n_max` that never happens, so a rejection is a snapshot-index rollback and not a state copy. **[V]**
+- If the build is older or the arch is not on the rollback list, the `COMMON_CONTEXT_SEQ_RM_TYPE_FULL` path copies a checkpoint every step (`use_ckpt_tgt`, [`#L3463`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L3463)); the startup line `speculative decoding will use checkpoints` ([`#L1388-L1390`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L1388)) would show it. Check the startup log for that line. **[V]** for the code, **[I]** for which case applies.
+- `--ctx-checkpoints` / `--checkpoint-min-step` (defaults 32 and 8192, [`common.h#L639-L642`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L639)) are for prompt/prefix reuse. Nothing I read ties them to the decode loop, so they should not cost decode time. **[I]**
+- The "cache_reuse is not supported by this context" startup line is the server's `llama_memory_can_shift` check ([`server-context.cpp#L1328-L1336`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L1328)). It confirms the open question in the throughput note. **[V]**
+
+## 3. Backend, HIP flags, clocks
+
+### 3.1 ROCm versus Vulkan (RADV)
+
+Everything here is **[R]** unless noted; the thread posts are self-reported and mixed in setup.
+
+| Source | Setup | Numbers |
+|---|---|---|
+| [discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043) (JohnTDI-cpu, build 8554) | R9700, RADV, Qwen3.5-27B Q4_K_M | tg128 29.07 t/s (RADV), 29.01 (AMDVLK); pp512 798 (RADV). Computed there as ~456 GB/s, 71 % of 640 GB/s; "best dense ~79-81 %" |
+| same | R9700, ROCm, 27B Q8_0, q4_0 KV, 64k ctx | pp 960 (3.7k) to 708 (58.7k) t/s; tg 42.8 to 28.0 t/s over the same range (MTP use not stated, and 42.8 t/s is above a 29 GiB Q8_0 plain-decode ceiling of ~22 t/s, so it is almost certainly with speculation) |
+| same | Windows, HIP, gfx1201 | pp512 1106-1211; tg128 29.07; "Vulkan decode still wins" |
+| same | R9700, Qwen3.6-27B UD-Q4_K_XL, q4_0 KV, MTP (build b9203) | tg128 28.2 baseline, 47.4 with `mtp-3` |
+| [discussion #20881](https://github.com/ggml-org/llama.cpp/discussions/20881) | R9700 and others | MoE Qwen3-Coder-30B-A3B Q4_0 ~196 t/s on RADV; one ROCm segfault report on Qwen3.5-35B-A3B |
+| [throughput note](qwen3.8-27b-throughput-tuning.md) (cat5edopeHA) | R9700, MTP n3 | Vulkan 68.5 / ROCm 55.1 short, 50.5 / 40.2 at 32k, 40.9 / 31.2 at 64k |
+
+- A search summary also cites a blog (not fetched) with ROCm ahead for dense 27B decode (42.8 vs 29.1), the opposite of cat5edopeHA. The sources conflict, so only a local A/B decides. **[R]**
+- DFlash2 on Vulkan: not verified. The DFlash graph and feature extraction live in common code and the model graph, so it may work, but I did not check that a Vulkan build runs `draft-dflash`. **[I]**
+- Prefill: HIP uses MMQ on RDNA4 unconditionally (`ggml_cuda_should_use_mmq` returns true for RDNA4, [`mmq.cu#L468-L471`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmq.cu#L468)), so `GGML_CUDA_FORCE_MMQ` changes nothing on gfx1201. **[V]**
+
+### 3.2 Decode kernels for the verify batch
+
+- Weights are Q4_K-family. Matrix multiplies with up to 8 columns go to MMVQ (`MMVQ_MAX_BATCH_SIZE 8`, [`mmvq.cuh`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmvq.cuh)). RDNA4 has no per-type override, so `ne11 <= 8` uses MMVQ ([`mmvq.cu#L318-L430`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmvq.cu#L318)). The source comment says k-quants "cost more to decode and mvq redoes that per column". **[V]** So the verify batch of 6 (n_max 5) or 8 (n_max 7) pays dequant ALU work per column; cost grows with batch width even though weight bytes are read once. **[I]** (the per-column cost on gfx1201 was not measured). This is the mechanism behind "lower n_max can win": it shrinks both the MMVQ columns and the drafter batch.
+- Batch 9 or more would fall to MMQ, which is not reachable: `n_max` is capped at 7 so the batch is at most 8. **[V]**
+
+### 3.3 HIP flags and environment variables
+
+| Item | Verdict | Source |
+|---|---|---|
+| `GGML_HIP_GRAPHS` | Already ON in the CMake default and the official ROCm Dockerfile does not turn it off. Variable graph shapes force a re-warmup: any change in node properties resets `warmup_complete` ([`ggml-cuda.cu#L4806-L4845`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/ggml-cuda.cu#L4806)). With `p_min > 0` the verify batch width varies, so graphs may be re-captured more often. Debug with `GGML_CUDA_GRAPH_DEBUG=1`; disable with `GGML_CUDA_DISABLE_GRAPHS=1`. | [V] ([`ggml/CMakeLists.txt#L217`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/CMakeLists.txt#L217), [`rocm.Dockerfile`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/.devops/rocm.Dockerfile)); effect on speed **[I]** |
+| `GGML_CUDA_FORCE_MMQ` | No effect on RDNA4 (see 3.1) | [V] |
+| `GGML_CUDA_GRAPH_OPT=1` | Opt-in multi-stream concurrent branches ([`ggml-cuda.cu#L5016-L5030`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/ggml-cuda.cu#L5016)). Off by default | [V] |
+| `GPU_MAX_HW_QUEUES=1` | HIP caps hardware queues per process per device (default 4, [HIP env docs](https://rocm.docs.amd.com/projects/HIP/en/docs-7.2.3/reference/env_variables.html)). llama.cpp issues all kernels on one stream unless `GGML_CUDA_GRAPH_OPT=1`, so one queue should cost nothing for decode. The repo set it for [ROCm/ROCm#5706](https://github.com/ROCm/ROCm/issues/5706) (`docs/architecture.md#gpu-access`). Do not combine `GGML_CUDA_GRAPH_OPT=1` with it without a test | [R] for the doc, [V] for the llama.cpp side, **[I]** for "no decode cost" |
+| `-bs, --backend-sampling` | Experimental, default off ([`arg.cpp#L2310`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L2310)). Moves target sampling onto the GPU, which would remove the CPU sampling of 6 logit rows per step. Interplay with speculation is unverified | [V] flag exists; benefit **[I]** |
+| `rocm-smi` perf level / compute profile | In #21043: "ROCm `rocm-smi` high vs auto: auto was faster" on a single card; a LACT 330 W cap with COMPUTE profile gave tg 102 to 110 on a dual-card MoE run; `power_dpm_force_performance_level=high` gave +14 % tg on one Vulkan run. Mixed, hardware and backend differ | [R] |
+| PCIe ASPM `pcie_aspm.policy=performance` | Same thread: dense decode on RADV 29.30 to 32.46 (+10.8 %), one user, not reproduced on eGPU or dual-slot. It is a runtime sysfs switch, so it is cheap to try | [R] |
+| MCLK not reaching max under load | Same thread, kernel 6.17, RADV | [R]; check with `rocm-smi` during decode, **[I]** |
+
+### 3.4 Threads, mmap, mlock, poll
+
+- With every layer on the GPU the CPU is only a scheduler. #21043 tested `-t`, `-b`, `-mmp 0`, `--prio 2`, CPU pinning, hugepages and `nice` on RADV and found all within +-0.3 % **[R]**. The server default `--poll` level is 50 ([`common.h#L77`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L77)) and `-t` was -1 (16 here). I found no evidence these matter for a GPU-resident run, so do not spend experiments on them. **[I]**
+- One exception worth knowing: speculative drafting has its own thread/poll flags (`--spec-draft-threads`, `--spec-draft-poll`, [`arg.cpp`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp)), but the drafter is on the GPU too, so they should be equally inert. **[I]**
+
+## 4. Quant and model options
+
+Inputs: GGUF sizes from [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) (UD-Q4_K_XL 17.6 GB, UD-Q4_K_M 16.5, UD-Q4_K_S 15.4) and the throughput note (UD-IQ4_XS 13.3 GB). Bandwidth 640 GB/s is the card spec as quoted in #21043 **[R]**. Weights only; 52k q8_0 KV adds 1.81 GB (2.1).
+
+| Quant | Bytes read/token | Ceiling at 640 GB/s | At 80 % efficiency | Same with 52k q8_0 KV (ceiling / 80 %) |
+|---|---|---|---|---|
+| UD-Q4_K_XL (running) | 17.6 GB | 36.4 t/s (27.5 ms) | 29.1 | 33.0 / 26.4 |
+| UD-Q4_K_M | 16.5 GB | 38.8 | 31.0 | 35.0 / 28.0 |
+| UD-Q4_K_S | 15.4 GB | 41.6 | 33.2 | 37.2 / 29.8 |
+| UD-IQ4_XS | 13.3 GB | 48.1 | 38.5 | 43.5 / 34.8 |
+
+- All **[I]** (arithmetic). The 80 % factor is the "best dense ~79-81 %" figure in #21043 **[R]**. The measured plain-decode rate at depth, 26-28 t/s, sits on the 52k column for Q4_K_XL (26.4), so the card is already near its bandwidth limit for plain decode. There is no hidden 2x in the non-speculative path.
+- A smaller quant gains at most ~13 % (Q4_K_M) to ~32 % (IQ4_XS) on the plain-decode ceiling and also shrinks the spec verify weight read. Quality cost is not measured here. The IQ4 family dequantizes slower than Q4_K, so the real gain is below the ceiling ratio. **[I]** Test with KL divergence or perplexity on a small fixed corpus (`llama-perplexity`) before any switch.
+- Drafter quant: Q4_K_M at ~1.1 GB is already the smallest tested, with acceptance as good as BF16 (1.4). Nothing to gain. **[R]**
+- Per spec step the drafter adds ~1.1 GB of reads (~1.7 ms at the ceiling), small against the ~45 ms fixed overhead found in 1.5. **[I]**
+
+## 5. Where the step time goes (open)
+
+Old-log step times are 74 ms at <8k and 94 ms at 40-60k ([1.5](#15-what-the-old-log-shows-re-analysis)). The weight-streaming floor at 80 % efficiency is ~34 ms. The remaining ~40-60 ms per step is not explained by any single item I could verify. Candidate contributors, none measured:
+
+1. 6-column MMVQ with k-quant dequant per column (3.2).
+2. Drafter forward over `n_max + 1` positions plus injecting 5 layers of target features per accepted token (`n_extract=5`, [`speculative.cpp#L1040`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1040)), including the encoder fused into KV injection ([commit 662a0b0](https://github.com/ggml-org/llama.cpp/commit/662a0b012)).
+3. CPU sampling of 6 full-vocab logit rows plus device-to-host copies per step (`-bs` is off).
+4. Host synchronisation between drafter and target per step (two contexts, no overlap).
+5. Quantized-to-f16 KV conversion for the TILE kernel at depth (2.2).
+
+Experiment C isolates 1 and 5 without the server; the rest need a profile (`rocprofv3`), which is out of scope here.
+
+## 6. Experiment plan
+
+Rules: one change per run, cold slot, same prompt, restart the server between configs with `./scripts/update.sh` on the server (this box cannot deploy). All numbers come from the server log lines `prompt eval time`, `eval time` and `draft acceptance` (`llmserver.log:32-36` show the format).
+
+### Benchmark protocol (minimal, comparable)
+
+1. **Fixed prompts** (kept in the repo's scratch area, not committed): P-code = ~8k tokens of source plus the instruction "continue the file" and P-code-52k = the same material padded to ~52k tokens; P-reason = a reasoning question over the same 52k context. Same bytes every run.
+2. **Fixed request:** `n_predict=512` (400+ tokens so the 3-s `tg` window averages out), `seed` fixed, `cache_prompt` false for the first request so prefill is cold, then the same request again with the prompt cache warm only if you want decode in isolation.
+3. **Sampler:** run each prompt twice, once with the production client settings (record them from the request body) and once with `temperature=0`. Record the sampler fields in the run name.
+4. **Repeats:** N = 5 per cell, both slots idle except the request under test; discard the first.
+5. **Read** from the log for each request: prompt tokens/s (`prompt eval time`), decode t/s (`eval time`), `draft acceptance` and `mean len`. Report median and min/max, not the mean.
+6. **Decode-only control:** `llama-bench` inside a throwaway container (stop `llama-server` first, one GPU): `llama-bench -m <model> -ngl 99 -fa 1 -ctk q8_0 -ctv q8_0 -p 0 -n 256 -d 0,16384,52000 -r 3`. `-n 256` avoids the "implicit ubatch 128 on hybrid models" artifact reported with `-n 128` in #21043 **[R]**. `llama-bench` does not use speculation or the prompt cache.
+7. **Environment capture** per run: `rocm-smi --showclocks --showpower --showtemp` once during the decode (SCLK, MCLK), plus the full startup log (build string, `n_rs_seq`, checkpoint line).
+
+### Experiments, in priority order
+
+| # | Change (exact) | Expected effect | How to measure | Risk |
+|---|---|---|---|---|
+| A | `LLAMA_SPEC_N_MAX` = 3, 4, 5, 7 (one run each); then 5 with `--spec-draft-p-min 0.3` and `0.5` | Fewer wasted drafted tokens and a narrower verify batch when acceptance is ~0.4; could be neutral or negative when acceptance is high. Direction at 52k is the point; short context favors the larger value. No estimate of size without data | Decode t/s, `mean len`, `generated` (must be a multiple of the n_max, or not if p_min is on), on P-code and P-code-52k | Low. `p_min` makes batch widths vary, which can trigger graph re-capture (3.3); if so, compare with `GGML_CUDA_DISABLE_GRAPHS=1` |
+| B | No server change: send P-code-52k / P-reason with (1) production sampler, (2) `temperature=0`, (3) production minus penalties, (4) production minus top-p/top-k | Isolates how much acceptance the sampler costs (6, 1.3). A large gap says the fix is in the client/gateway sampler defaults, not llama.cpp | `draft acceptance` and `mean len` per cell; the full per-position `acc per pos` appears at trace level only | None |
+| C | Stop `llama-server`; in a container with the same image: `llama-bench ... -p 1,2,4,6,8 -n 0 -d 0,52000` and `-p 0 -n 256 -d 0,52000`; rocm-smi clocks during a server decode | Splits step time into verify-batch cost (pp6 time at depth) vs plain tg1. Shows whether verify at depth already costs ~60+ ms | `llama-bench` t/s table; compare pp6 time with measured step time | Low. Needs a short outage of the only GPU consumer |
+| D | `--parallel 1` (then `--no-kv-unified` with `--parallel 2`) on the same P-code-52k | Tells whether the second slot or the unified cache costs single-stream decode (2.3). If decode at 52k jumps, slot layout is the cost | Decode t/s on a request with the other slot idle; then with the other slot holding a 50k context | Medium. `--parallel 1` serialises agents; non-unified halves per-slot max context |
+| E | `echo performance \| sudo tee /sys/module/pcie_aspm/parameters/policy`, and check MCLK/SCLK under load; try `rocm-smi --setperflevel high` only if MCLK sags | Reported +10.8 % on one RADV dense run; unknown for ROCm. Free to try and revert | Decode t/s on P-code (plain control from C) | Low; revert with the previous policy value |
+| F | `-bs` (`--backend-sampling`) | Removes CPU sampling of the verify rows if that is a real share of the step | A/B on P-code | Medium: experimental, speculation interplay unverified |
+| G | `GGML_CUDA_DISABLE_GRAPHS=1` and, separately, `GGML_CUDA_GRAPH_DEBUG=1` to see re-warmup counts | Tells whether graph churn from the variable shapes is a cost | A/B on P-code | Low |
+| H | `-ctk q4_0 -ctv q4_0` at 52k | Saves ~0.85 GB read per token (~4-5 % on plain decode) and removes some f16 conversion traffic; quality risk | A/B on P-code-52k plus a spot-check of output quality | Medium (quality) |
+| I | `-ctk f16 -ctv f16 --ctx-size 131072` | Removes q8_0 to f16 conversion for the TILE verify kernel (2.2); cache is 1.9x larger. VRAM: f16 KV for 131072 tokens is ~8 GB | A/B on P-code-52k | Medium (halves context, VRAM) |
+| J | Smaller quant (UD-Q4_K_M or UD-IQ4_XS) from the same repo | Up to +13 % / +32 % on the plain ceiling (4); quality unmeasured | A/B plus KLD/perplexity | Medium (quality, download) |
+| K | Vulkan (RADV) image, if `draft-dflash` runs on it | Reported decode lead on some setups, but conflicting sources and prefill lag at short context | A/B with the same protocol | High (new image, unverified DFlash2 support) |
+| L | `--spec-type draft-dflash,ngram-mod` (list is accepted, [`arg.cpp#L4314-L4322`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4314)) | Extra drafts on repeated text (tool JSON, code). Interaction with the DFlash drafter is unchecked | A/B on P-code | Medium |
+
+Not worth an experiment: `-t`, `--poll`, `--no-mmap`, `--mlock`, `GGML_CUDA_FORCE_MMQ`, drafter quant (3.4, 3.1, 1.4).
+
+## Open questions
+
+- What does the server `.env` say for `LLAMA_SPEC_N_MAX`, and what does the new startup log print for `speculative decoding will use checkpoints` and `n_rs_seq`?
+- What temperature, top-p/top-k and penalties do OmniRoute clients actually send? If they differ per client, acceptance will differ per client.
+- Is the ~45 ms fixed overhead per spec step in the target verify, the drafter, or the CPU? Experiment C and a `rocprofv3` trace would answer it.
+- Does a Vulkan build run `draft-dflash` at all?
+- Which exact llama.cpp build is in the image, and is it before or after the `n_rs_seq` snapshot change?
+
+## Sources
+
+- Repo: `llmserver.log` (lines cited inline), `docker-compose.yml`, `.env.example`, `docs/architecture.md#llama-server`, `docs/research/qwen3.8-27b-throughput-tuning.md`, `docs/research/qwen3.8-27b-quant.md`.
+- llama.cpp at `839a572a4cdecdb80d211297b8e7d7323a6e99d2`: `common/speculative.cpp`, `common/arg.cpp`, `common/common.h`, `common/sampling.cpp`, `tools/server/server-context.cpp`, `src/llama-kv-cache.cpp`, `src/llama-arch.cpp`, `src/llama-context.cpp`, `src/llama-memory-recurrent.cpp`, `ggml/src/ggml-cuda/{fattn.cu,fattn-common.cuh,mmq.cu,mmvq.cu,mmvq.cuh,ggml-cuda.cu}`, `ggml/CMakeLists.txt`, `.devops/rocm.Dockerfile`. Line numbers were read from `raw.githubusercontent.com/.../master/...` on the day of fetch; if the link anchors drift, search by the quoted identifier.
+- [PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) (DFlash2), [issue #27117](https://github.com/ggml-org/llama.cpp/issues/27117), [discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043), [discussion #20881](https://github.com/ggml-org/llama.cpp/discussions/20881).
+- [z-lab/Qwen3.8-27B-DFlash2-GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF), [z-lab/Qwen3.8-27B-DFlash2](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2) (via a search summary), [HIP environment variables](https://rocm.docs.amd.com/projects/HIP/en/docs-7.2.3/reference/env_variables.html).
+- Not fetched: the Phoronix R9700 ROCm vs Vulkan review (HTTP 403), AMD ROCm RDNA4 pages beyond the HIP env page.

docs/research/qwen3.8-27b-throughput-tuning.md

@@ -0,0 +1,140 @@
+# Research: faster tokens/s on the R9700 (llama-server, Qwen3.8-27B + DFlash2)
+
+**Date:** 2026-10-20
+
+**Question:** Can `llama-server`'s configuration be tuned for more tokens per second, and can system DDR RAM act as a slower cache tier? Evidence: `llmserver.log` (about 3 h of traffic) plus five outside reference configs.
+
+**Scope note:** research only. Nothing in compose or `.env.example` was changed.
+
+## TL;DR, ranked
+
+| # | Change | Expected gain | Risk | Confidence |
+|---|---|---|---|---|
+| 1 | Find out why ~half of all requests are full ~40-50k-token re-prefills (log shows `selected slot by LRU` and no prefix reuse), then fix the prefix instability or give the cache room: `--cache-ram` up (e.g. 24576), keep `--ctx-checkpoints` at its default | Largest available. Prefill is 2.06 h of the ~4.7 h of summed slot time in the log. Every avoided 45k prefill saves ~47 s | Low (host RAM only) | Medium: the cause is not visible in the log (see Open questions) |
+| 2 | Sweep `--spec-draft-n-max` (try 3, 4, 5 vs 7) and add `--spec-draft-p-min` (e.g. 0.5) | Maybe +5 to +15 % decode, unmeasured. Measured acceptance is only 50 % at n_max 7 | Low | Low-medium: needs an A/B |
+| 3 | A/B `-ub 1024` or `2048` (with `-b 2048`+) for prefill | Maybe +5 to +20 % prefill, unmeasured. One reference saw no gain on Vulkan, another uses 8192/8192 | Low-medium (compute buffer VRAM; ~6 GB headroom) | Low |
+| 4 | A/B the Vulkan (RADV) build against ROCm | A reference measured +23 to +31 % decode with MTP, and faster prefill at 32k+ depth. ROCm was faster at short prompts | Medium (new image, DFlash2 support on Vulkan is unchecked) | Medium (single third-party repo) |
+| 5 | Do NOT expect gains from: `--no-kv-offload`, `--n-cpu-moe`/`--cpu-moe`, `--mlock`, RAM layer offload | None or negative | n/a | High |
+
+Decode (45.9 t/s weighted) is already well above plain decode (~25-30 t/s for this model on this card per the references), so DFlash2 works. The remaining headroom is in prefill and in cache hits, not raw decode.
+
+## Measured baseline (from `llmserver.log`)
+
+**The log is a mid-run tail. It has no startup block.** There is no build string, `n_ctx`, KV buffer sizes, ubatch or device memory line, so the llama.cpp version and the values below come from compose, not from the log. Grep for `build:`, `load_tensors`, `KV buffer`, `context checkpoints enabled` and `cache_reuse is not supported` returned nothing. Get those lines with `docker logs llama-server | head -200` on the server after the next restart. The timestamp format is `MMMMM.SS.mmm.uuu` (minutes of uptime), so the 10917 to 11098 range is about 181 min.
+
+All figures below were computed over the whole file (10000 lines).
+
+- Tasks: 360 launched (`launch_slot_`), 305 completed with a timing summary, 56 `cancel task` lines (first at `llmserver.log:2113`).
+- **Generation:** weighted average 45.9 t/s over 380k generated tokens. Per request (those with >=200 tokens): average 49.4, max 76.9, min 4.1 t/s. Running `tg` (the live counter, `llmserver.log:98`-`100`): p10 15.0, p50 33.0, p90 56.6 t/s. The low end is decode competing with the other slot's prefill, not a different config.
+- **Prefill:** 933-980 t/s on 38-50k-token prompts (`:32`, `:65`). Rate falls with depth, from 1282 t/s at the first 6k tokens to ~945 t/s at 50k (`:8`-`:31`). Bucket averages: <500 tokens 417 t/s, 0.5-5k 839, 5-20k 952, >20k 904.
+- **Speculation:** draft acceptance 50.1 % overall (n=306 summaries), mean draft length 3.86. Individual requests range 0.38 (`:36`) to 0.71 (`:69`). n_max is 7, so a lot of drafted tokens are rejected.
+- **Totals:** summed prefill time 7404 s, summed decode time 9503 s. Prefill is not the minority.
+- **Request shape:** 146 of 305 requests were >20k-token prefills (average 42.5k tokens, ~47 s each). Typical output on those is small (`:33` 99 tokens, `:66` 78 tokens). `release` n_tokens: min 1097, median 8395, max 57073.
+- **Cache behaviour:** `--cache-reuse` is evidently not what saves these requests. 200 slot selections were `by LRU`, 160 `by LCP similarity` (average f_sim_best 0.60, e.g. 0.997 at `:2279`). 253 `making room for prompt cache entry, removing oldest entry (size ~1.8-2.3 GiB)` warnings (`:7`, `:40`, `:73`) show the host-RAM prompt cache is active and full. The upstream default is 8192 MiB (see sources), so it holds only ~3-4 such entries.
+- **Pattern at `:380`-`:470`:** requests alternate slot 0 / slot 1 and each does a full ~38-46k prefill by LRU. The prefix is not shared with the previous request on either slot.
+
+Configured (compose, `docker-compose.yml`): `--ctx-size 262144`, `--parallel 2` (131072 per slot), `--kv-unified`, `--flash-attn on`, `--cache-type-k/v q8_0`, `--cache-reuse 256`, `--spec-type draft-dflash`, `-md ...DFlash2-Q4_K_M.gguf`, `--spec-draft-n-max 7`, `--n-gpu-layers 999`, `GPU_MAX_HW_QUEUES=1`. Not set, so defaults: `-b 2048`, `-ub 512`, threads, `--cache-ram` (8192), `--ctx-checkpoints` (32), `--spec-draft-p-min` (0.00), sampling for the drafter. Prior measured VRAM: ~81 % (25.8 of 31.9 GB) at 262144 q8_0 (project memory note `llama-server-vram-baseline`).
+
+## Flag verification (primary: llama.cpp master)
+
+Sources: `tools/server/README.md` and `common/arg.cpp` and `tools/server/server-context.cpp` on `ggml-org/llama.cpp` master, fetched 2026-10-20. Master is not necessarily the build in the log (version unknown, see above), so check `llama-server --help` inside the container before using any flag.
+
+| Flag | Exists on master | Default / note |
+|---|---|---|
+| `-cram, --cache-ram N` | yes | default 8192 MiB, -1 no limit, 0 disable. Env `LLAMA_ARG_CACHE_RAM` |
+| `-ctxcp, --ctx-checkpoints N` (alias `--swa-checkpoints`) | yes | default 32 per slot |
+| `-cms, --checkpoint-min-step N` | yes (server README) | default 8192 tokens, 0 = no minimum |
+| `--cache-reuse N` | yes | "min chunk size to attempt reusing from the cache via KV shifting", default 0. `server-context.cpp` disables it with the log line `cache_reuse is not supported by this context, it will be disabled` if `llama_memory_can_shift` is false |
+| `--cache-idle-slots` | yes | default enabled, requires cache-ram |
+| `-sps, --slot-prompt-similarity` | yes | default 0.10 (matches the `0.100 thold` in the log) |
+| `-b`, `-ub` | yes | 2048 / 512 |
+| `--spec-type`, `--spec-draft-n-max/-n-min/-p-min` | yes | n_max default 3, n_min 0, p_min 0.00. The README says the old `--draft-max` "has been removed" |
+| `-kvo/-nkvo, --kv-offload` | yes | default enabled |
+| `-lm, --load-mode` (mmap, mlock, ...) | yes | default auto. Newer master has no standalone `--mlock`/`--no-mmap` in the README text I read; older builds do |
+| `-cmoe`, `-ncmoe` | yes | MoE only. Irrelevant for a dense model |
+| `-t, --threads` | yes | -1 |
+
+Not verified: that the `draft-dflash` value of `--spec-type` is in the log's build (the stack evidently runs it, so it is). Not verified: that hybrid (Gated DeltaNet + attention) memory reports `can_shift = false`. I did not read `llama-memory-hybrid.cpp`. If it does, `--cache-reuse 256` is silently disabled for this model and the real mechanism is checkpoints plus the prompt cache. Check the startup log for the `cache_reuse is not supported` line. Also not verified: the exact condition under which a checkpoint is created (the fetched `server-context.cpp` was cut off before it), so the checkpoint advice below is a lead.
+
+### Recommendation details
+
+1. **Prompt cache / re-prefill (rank 1).** The cost is in the log, the cause is not. Three candidates, in order of likelihood:
+   - (a) the client or gateway changes bytes early in the prompt on each request (timestamps, session ids, memory injection, tool list order), so LCP is near zero. Fix at the source: diff two consecutive prompts from a long gap.
+   - (b) the model is hybrid, a recurrent state cannot be rolled back to an arbitrary token, so partial-prefix reuse needs a checkpoint at the right position (`--ctx-checkpoints`, `--checkpoint-min-step`). With `-cms` 8192, a checkpoint sits at most every 8192 tokens, so some prefix is lost on each divergence. `-cms 2048` is a candidate, at ~tens of MiB each (unmeasured).
+   - (c) the cache is too small. 2.3 GiB per 50k-token entry against 8192 MiB means ~3 entries for 2 slots plus history. Raise `--cache-ram` (host RAM). This is the DDR RAM tier, see below.
+2. **Speculation (rank 2).** Acceptance 50 % at n_max 7 means expected accepted length ~3.9. The z-lab/lucebox write-up notes DFlash2 was trained at block 8 and that `--spec-draft-n-max 7` is clamped to it, so 7 is already the maximum useful value. Lower n_max cuts wasted drafter and verify work at the cost of fewer tokens per step. `--spec-draft-p-min` makes the drafter stop early on low-confidence tokens. Both are standard knobs, but no reference tuned them for DFlash2 on llama.cpp. Treat as an A/B, not a recommendation. Greedy sampling collapses DFlash2 acceptance (cat5edopeHA `ISSUES.md` #8), so if clients send temperature 0 for some workloads, acceptance will be bad there. Check per-request temperature against the 0.38-0.71 spread.
+3. **Batch (rank 3).** Prefill rate is 933-980 t/s at 40-50k depth. References on the same card: cat5edopeHA nospec ROCm 541 t/s at 32k and 298 at 64k (Q4_K_M, `-b/-ub` default), rahlquist 727 at 40k and 564 at 65k with 8192/8192 (f16 KV, `llama-bench`). This repo's 945 at 46k is already faster than either at that depth, so do not expect large batch gains. Only one controlled `-ub` data point exists (cat5edopeHA, Vulkan, decode got slightly worse with 1024/1024).
+4. **KV quant (neutral).** q8_0 keeps VRAM headroom. lucebox measured f16 KV slightly slower than q8_0 on its own engine (202.5 vs 208.1 t/s). Do not drop to q4_0 for speed. rahlquist serves q4_0 but gives no quality data.
+5. **Threads, mmap, mlock.** The whole model and KV sit in VRAM, so CPU thread count affects only the host-side sampling/scheduling. No gain expected. `--mlock` would pin the 17.6 GB GGUF in RAM for no benefit once it is on the GPU.
+
+## DDR RAM as a slower tier
+
+| Idea | Helps t/s? | Why |
+|---|---|---|
+| Layer offload to CPU (`--n-gpu-layers` below 999) | No, slower | Dense decode reads all weights per token. VRAM bandwidth is several times DDR. Every layer moved to RAM adds PCIe/DDR-bound time. Weights (17.6 GB) already fit |
+| `--cpu-moe`, `--n-cpu-moe` | No | Model is dense. Flag text is "keep the Mixture of Experts (MoE) weights ... in the CPU" |
+| `--no-kv-offload` (KV in RAM) | No, much slower | Attention reads KV every step. Keeps VRAM but moves the hot path to DDR |
+| **`--cache-ram` (host-RAM prompt cache)** | **Yes, for agent workloads** | It is exactly this feature. It stores whole-slot states in host RAM and reloads them instead of re-prefilling (`get_available_slot` saves a slot when `f_keep < 0.5`, per `server-context.cpp`). The log shows it already churning at the 8192 MiB default |
+| `--slot-save-path` | Maybe, manual | Persists slots to disk, restore via API. Needs client orchestration |
+| mmap | Only a load-time effect | Model is on GPU after load |
+
+Rough sizing: entries are 1.8-2.3 GiB for 38-50k tokens (log `:7`, `:40`, `:73`). 32 GiB of cache would hold ~14 such entries. Host RAM size on the server is not known to me. Restoring an entry copies about 2 GiB over PCIe, on the order of a second, against ~47 s of prefill. That timing is an estimate, not measured.
+
+## Comparison with reference configs
+
+Legend: **V** = independently verified against llama.cpp source or reproduced in this repo's own data. **P** = reported only by the page, treated as a lead.
+
+| Source | Quant / engine | Ctx, KV | Spec | Batch | R9700 numbers | Status |
+|---|---|---|---|---|---|---|
+| **This repo (log)** | UD-Q4_K_XL, llama.cpp ROCm | 262144, q8_0, `-np 2` | DFlash2 n_max 7 | default 2048/512 | decode 45.9 t/s avg, prefill 933-980 t/s at 40-50k | V (log) |
+| [cat5edopeHA/qwen38-27b-ai1](https://github.com/cat5edopeHA/qwen38-27b-ai1) (first-party repo) | Q4_K_M + baked MTP, llama.cpp `d222767`, HIP and Vulkan | 262144, q8_0, `-np 1`, `-fa on`, `-ngl 99` | `draft-mtp` n_max 3 (best on both; Vulkan 4-5 wins at >=8k) | defaults; `-b 1024 -ub 1024` no gain | mtp3 decode Vulkan/ROCm: 68.5/55.1 short, 50.5/40.2 at 32k, 40.9/31.2 at 64k. nospec 25-30. Prefill ROCm 1047 short, 541 at 32k, 298 at 64k (Vulkan 377 at 64k) | P for numbers (not rerun). Its flags exist on master (V) |
+| [rahlquist/qwen38-r9700](https://github.com/rahlquist/qwen38-r9700) (first-party repo) | Q4_K_M, ROCm build 10438 | 65536, f16 bench, q4_0 in serving | MTP draft did not load (VRAM) | 8192/8192 (ubatch 512 at 128k) | prefill 1080 at 2.5k, 727 at 40k, 564 at 65k. Decode 26.4 to 22.5 (no spec) | P. Its decode matches nospec numbers elsewhere |
+| [lucebox blog](https://www.lucebox.com/blog/qwen38-r9700) (secondary) | UD-IQ4_XS, own `dflash_server` engine (not llama.cpp); llama.cpp + PR #27342 for comparison | 131072, q8_0 | DFlash2 `--draft-block-size 16`; llama.cpp run used n_max 7 | not stated | own engine 208 decode (HumanEval, greedy). **llama.cpp + DFlash2: 54.6 decode**, 59.7 at 1.5k, ~31-35 at 36-70k | P. The llama.cpp row is the fair comparison to this repo: log average 45.9 sits inside it |
+| [aifactum.de](https://aifactum.de/ki-hardware/qwen3-8-27b-lokal-auf-amd-r9700/) (secondary) | llama.cpp Vulkan, Windows (AMD figure, preliminary) | not stated | MTP, 2 draft tokens | not stated | up to 51.8 t/s; advises disabling mmap in LM Studio | P. Vendor number, no config |
+| [willitrunai.com](https://willitrunai.com/can-run/qwen-3.8-27b-on-radeon-ai-pro-r9700-32gb) (secondary) | Q4_K_M, estimator | 168K "safe" | none | none | decode 16.3, prefill 688 (all estimates) | P. A calculator, contradicts itself (22.7 vs 16.9 GB) and is below measured data. Ignore |
+
+Take-aways:
+
+- Same card, no-spec decode is ~25-30 t/s in three independent sources. This repo's 45.9 t/s with DFlash2 is in line with the lucebox llama.cpp + DFlash2 row (54.6 on short HumanEval prompts, ~31-36 at deep context). **No reference shows a materially faster llama.cpp ROCm decode than this repo already has.** The 200+ t/s figure is a different engine (lucebox) with a 16-wide block and greedy short prompts, not a flag change here.
+- The only reference-backed decode lever is the **backend**: Vulkan decode beat ROCm by 23-31 % in one repo, but ROCm was faster at prefill for short prompts, and Vulkan faster at >=32k. This workload is both prefill-heavy and deep, so the net is unclear. Needs a local `llama-bench` on both builds. The repo's `docs/research/qwen3.8-27b-quant.md` already mentions the same Vulkan claim from a community source.
+- Differences that do not obviously matter: this repo's quant (UD-Q4_K_XL, 17.6 GB) is bigger than UD-IQ4_XS (13.3 GB) and Q4_K_M (16.5 GB). A smaller quant reads fewer bytes per token and would decode a little faster, at a quality cost not measured here. lucebox's `--pure` requant claim is about their own engine and not applicable.
+- cat5edopeHA used `-np 1`. This repo uses 2, and the log shows decode slowing while the other slot prefills (running tg p10 15 t/s vs p90 57 t/s). `-np 1` removes that interference but serializes agents and risks the gateway timeouts documented in `docs/gateway.md`.
+
+## Proposed diffs (not applied)
+
+All via `.env`/compose, A/B one at a time. Compose is the source of truth, so these would need doc updates per `docs/agents/writing-docs.md` rule 4.
+
+```diff
+ # docker-compose.yml, llama-server command
++      --cache-ram ${LLAMA_CACHE_RAM:-24576}      # rank 1, host RAM
++      --checkpoint-min-step ${LLAMA_CHECKPOINT_MIN_STEP:-2048}   # rank 1 lead, verify flag name on the running build
+-      --spec-draft-n-max ${LLAMA_SPEC_N_MAX:-7}
++      --spec-draft-n-max ${LLAMA_SPEC_N_MAX:-7}  # sweep 3,4,5,7
++      --spec-draft-p-min 0.5                     # rank 2, A/B
++      -ub 1024                                   # rank 3, A/B (-b stays 2048)
+```
+
+Before any of that, confirm VRAM headroom: ~6 GB was left at 81 % (memory note). A larger `-ub` raises the compute buffer.
+
+## How to benchmark before and after
+
+1. **Baseline from the server's own timings.** Replay a fixed ~40k-token prompt and a fixed ~2k-token prompt, `temperature` as production uses (not 0), 3 repeats, one slot idle. Read `timings.prompt_per_second`, `predicted_per_second`, `draft_n`, `draft_n_accepted` from the response (the field names are used by cat5edopeHA's harness; confirm on this build).
+2. **`llama-bench`** for batch sweeps, inside the container (build must match): `llama-bench -m <model> -ngl 99 -fa 1 -ctk q8_0 -ctv q8_0 -p 512,8192,40000 -n 128 -ub 512,1024,2048`. `llama-bench` does not exercise speculation or the prompt cache, so it only informs rank 3 and the backend question.
+3. **Cache hit rate.** After changes, count `selected slot by LCP similarity` vs `by LRU` and `making room for prompt cache entry` per hour in the log, and sum `prompt eval time`. Target: sum of prefill seconds per hour drops. This is the metric for rank 1.
+4. Change one variable per run. Restart `llama-server` with `./scripts/update.sh` on the server (this box cannot deploy).
+
+## Open questions
+
+- Why do consecutive ~45k-token requests share no prefix? Capture two raw request bodies and diff the first 2k tokens. Check whether OmniRoute's slot affinity is active (log shows LRU selection 200 times).
+- What llama.cpp build is running, and does the startup log say `cache_reuse is not supported by this context`? If yes, `--cache-reuse 256` can be removed and `docs/architecture.md` is wrong about it.
+- How much host RAM does the server have free for `--cache-ram`?
+- Do clients send temperature 0 (collapses DFlash2 acceptance per cat5edopeHA `ISSUES.md` #8)? Acceptance varies 0.38-0.71 per request.
+- Is DFlash2 supported on a Vulkan llama.cpp build (the reference used a fork for it)? Unchecked.
+- Which cancellations (56 `cancel task`) are client timeouts mid-prefill, and do they waste prefill work?
+
+## Sources
+
+- Log facts: `g:\_DEV\repos\LLM-Server\llmserver.log` (line numbers above).
+- Compose and docs: `docker-compose.yml`, `.env.example`, `docs/architecture.md#llama-server`, `docs/research/omniroute-non-ping-sse-stream-timeout.md`, `docs/research/qwen3.8-27b-quant.md`.
+- llama.cpp primary: [tools/server/README.md](https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/server/README.md), [common/arg.cpp](https://raw.githubusercontent.com/ggml-org/llama.cpp/master/common/arg.cpp), [tools/server/server-context.cpp](https://raw.githubusercontent.com/ggml-org/llama.cpp/master/tools/server/server-context.cpp), [PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) (cited by the repo's own docs, not re-read here).
+- References: [cat5edopeHA/qwen38-27b-ai1](https://github.com/cat5edopeHA/qwen38-27b-ai1) (README, RECIPES.md, ISSUES.md, BENCHMARKS.md), [rahlquist/qwen38-r9700](https://github.com/rahlquist/qwen38-r9700), [lucebox](https://www.lucebox.com/blog/qwen38-r9700), [aifactum.de](https://aifactum.de/ki-hardware/qwen3-8-27b-lokal-auf-amd-r9700/), [willitrunai.com](https://willitrunai.com/can-run/qwen-3.8-27b-on-radeon-ai-pro-r9700-32gb). The WebFetch summaries of these pages were produced by a small model; numbers were not re-checked against the raw pages.

models.yml

@@ -0,0 +1,7 @@
+# Model files the downloader fetches into the models volume; see docs/operations.md#model-downloads
+# `file` must match the name in .env that loads it (LLAMA_MODEL_FILE, or -md in LLAMA_SPEC_ARGS).
+models:
+  - file: Qwen3.8-27B-UD-Q4_K_XL.gguf
+    url: https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-UD-Q4_K_XL.gguf
+  - file: Qwen3.8-27B-DFlash2-Q8_0.gguf
+    url: https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q8_0.gguf

scripts/bench.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Benchmarks llama-server with SPEED-Bench; see docs/operations.md#speed-bench
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+usage() {
+  echo "Usage: $0 run <name> [speed_bench args...]   # writes bench-results/<name>.json" >&2
+  echo "       $0 compare <baseline> <speculative>   # names from earlier runs" >&2
+  exit 1
+}
+
+case "${1:-}" in
+  run)
+    [ $# -ge 2 ] || usage
+    name=$2; shift 2
+    mkdir -p bench-results
+    docker compose --profile tools run --rm speed-bench --output "$name.json" "$@"
+    ;;
+  compare)
+    [ $# -eq 3 ] || usage
+    docker compose --profile tools run --rm --entrypoint python speed-bench \
+      /app/speed_bench_compare.py --baseline "$2.json" --speculative "$3.json"
+    ;;
+  *) usage ;;
+esac

scripts/redownload-model.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Forces a fresh download of a model file listed in models.yml, replacing the one on disk.
+# Why and how: docs/operations.md#forcing-a-model-re-download
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [ $# -gt 1 ] || [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
+  echo "Usage: $0 [model-file]   (default: LLAMA_MODEL_FILE from .env)" >&2
+  exit 1
+fi
+
+file="${1:-}"
+if [ -z "$file" ]; then
+  [ -f .env ] || { echo ".env missing, run scripts/update.sh first" >&2; exit 1; }
+  file="$(grep -m1 '^LLAMA_MODEL_FILE=' .env | cut -d= -f2- | tr -d '\r"'"'" || true)"
+  [ -n "$file" ] || { echo "LLAMA_MODEL_FILE is not set in .env" >&2; exit 1; }
+fi
+grep -qxF "  - file: $file" models.yml || { echo "$file is not listed in models.yml" >&2; exit 1; }
+
+# Runs a snippet against the models volume; $1 in it is the model file.
+in_volume() { docker compose --profile tools run --rm --entrypoint sh downloader -c "cd /models; $1" _ "$file"; }
+
+# Put the previous file back if the download didn't finish, so a failed run never loses the working model.
+ok=false
+restore() {
+  $ok || in_volume '[ ! -f "$1.old" ] || { rm -f "$1" "$1.part"; mv "$1.old" "$1"; }' || true
+}
+trap restore EXIT
+
+echo "==> moving aside the current $file (kept until the new download succeeds)"
+in_volume '[ -f "$1" ] || { [ ! -f "$1.old" ] || mv "$1.old" "$1"; }; [ ! -f "$1" ] || mv -f "$1" "$1.old"; rm -f "$1.part"'
+
+echo "==> downloading $file"
+docker compose --profile tools run --rm downloader
+
+ok=true
+in_volume 'rm -f "$1.old"'
+
+echo "==> restarting llama-server to load the new file"
+docker compose restart llama-server

scripts/switch-model.sh

@@ -1,50 +0,0 @@
-#!/usr/bin/env bash
-# Swap GPU residency between llama-server (Qwen) and comfyui — they never
-# run concurrently, VRAM doesn't fit both (see issue #38's map). Manual
-# invocation only, no auto-switching.
-#
-# Bypasses lazytainer entirely and drives docker compose directly — its
-# idle-stop can't be used for this. Root cause (see
-# docs/research/lazytainer-omniroute-idle-stop.md, issue #40): lazytainer's
-# packet-threshold detector is source-blind and can't tell OmniRoute's
-# periodic health-check pings apart from real traffic on the same port, so
-# it never reliably sleeps a service on its own. A scripted swap always
-# knows which service should go up/down, so it doesn't need that heuristic.
-#
-# OmniRoute may show the just-stopped provider as errored/offline in its
-# dashboard for up to CREDENTIAL_HEALTH_CHECK_INTERVAL (default 5 min) after
-# a swap — cosmetic, not a functional problem (see the research doc above).
-set -euo pipefail
-cd "$(dirname "$0")/.."
-
-usage() {
-  echo "Usage: $0 {qwen|comfyui}" >&2
-  echo "  qwen    - stop comfyui, start llama-server" >&2
-  echo "  comfyui - stop llama-server, start comfyui" >&2
-  exit 1
-}
-
-[ $# -eq 1 ] || usage
-
-case "$1" in
-  qwen)
-    from=comfyui
-    to=llama-server
-    ;;
-  comfyui)
-    from=llama-server
-    to=comfyui
-    ;;
-  *)
-    usage
-    ;;
-esac
-
-echo "==> stopping $from"
-docker compose stop "$from"
-
-echo "==> starting $to"
-docker compose up -d "$to"
-
-echo "==> status"
-docker compose ps

scripts/update.sh

@@ -1,28 +1,6 @@
 #!/usr/bin/env bash
-# The one command to run after any change to this repo (compose file, .env,
+# Brings the running stack in sync with this repo; safe to re-run any time.
-# or a git pull) to bring the running stack in sync. Ensures secrets/keys
+# What it does and why: docs/operations.md
-# exist, pulls, validates, rebuilds/re-pulls images, and recreates only what
-# changed — safe to run any time, including with nothing to do.
-#
-# Tunable config values (LLAMA_*, ports, timeouts — anything with a real
-# default in .env.example) are synced from .env.example every run. A value
-# already matching is left alone silently. A value that DIFFERS from the
-# server's current .env is a conflict: interactively, you're shown every
-# conflict on one screen (via gum) and choose which to accept — unpicked
-# keys keep the server's current value. Non-interactively (no TTY — cron,
-# CI, piped), any conflict is a hard error unless --force is passed, which
-# accepts every new value automatically. Secrets and host-resolved values
-# (blank in .env.example — OMNIROUTE_*_SECRET/_KEY/_SALT/_PASSWORD,
-# SEARXNG_LAN_IP, COMFYUI_PUID/PGID, HOST_VIDEO_GID/RENDER_GID) are never
-# touched by this — they keep going through set_if_blank as before.
-#
-# omniroute's own routing/provider config (llama-server, search) lives in
-# its dashboard, not a checked-in file like the old litellm-config.yaml —
-# see issue #31 and docs/proxy-key-onboarding.md.
-#
-# ponytail: no rollback/backup logic — this is a single-user homelab box,
-# not a fleet. If a bad config lands, `git revert` + re-run is the recovery
-# path, not this script.
 set -euo pipefail
 cd "$(dirname "$0")/.."
 
@@ -34,25 +12,14 @@ for arg in "$@"; do
   esac
 done
 
-# Must run before anything else touches a file this script itself reads
+# Keep this first — see docs/operations.md#self-update
-# (docker-compose.yml, .env.example, this script's own remaining lines) —
-# a self-updating script isn't guaranteed atomic against its own file
-# changing mid-run, so pulling later can execute a mix of old and new
-# script/compose content in one pass. Bit us for real: old GID-resolution
-# code ran, then this pulled in new var names docker-compose.yml now
-# requires, and nothing re-ran the (now-current) resolution step for
-# them — see issue #5's thread.
 echo "==> git pull"
 git pull --ff-only
 
 [ -f .env ] || cp .env.example .env
 
 echo "==> syncing tracked config values from .env.example"
-# ponytail: gum (charmbracelet/gum) is a single static binary. Vendored as
+# see docs/operations.md#gum
-# a release tarball under scripts/vendor/ (checked into git) for the R9700
-# box, which has no outbound internet access — the download fallback below
-# is only for other archs / when the vendored copy is missing or stale.
-# Cached under .cache/gum/ (gitignored) so repeat runs don't re-extract.
 GUM_VERSION="0.14.5"
 GUM_DIR="$(pwd)/.cache/gum"
 GUM_BIN="$GUM_DIR/gum"
@@ -84,10 +51,7 @@ ensure_gum() {
   [ -x "$GUM_BIN" ] && echo "$GUM_BIN" || echo ""
 }
 
-# Collect every key where .env.example has a real (non-blank) default:
+# see docs/operations.md#env-sync
-# missing from .env -> just add it (no conflict, nothing to decide);
-# present and identical -> leave alone silently; present and different ->
-# a conflict to resolve below.
 conflict_keys=()
 conflict_old=()
 conflict_new=()
@@ -128,8 +92,6 @@ if [ "${#conflict_keys[@]}" -gt 0 ]; then
     if [ -n "$gum_bin" ]; then
       selected="$(printf '%s\n' "${labels[@]}" | "$gum_bin" choose --no-limit --selected "$(printf '%s\n' "${labels[@]}" | paste -sd,)" --header "Config differs from .env.example — selected keys take the new value, unselected keep the server's current value:")"
     else
-      # ponytail: plain-bash fallback if gum couldn't be fetched (offline,
-      # unsupported arch) — same one-screen-of-conflicts idea, cruder UI.
       echo "Config differs from .env.example. Enter space-separated numbers to KEEP the server's current value (all others take the new value), or press enter to take every new value:"
       for i in "${!conflict_keys[@]}"; do
         echo "  $((i+1))) ${labels[$i]}"
@@ -155,9 +117,6 @@ if [ "${#conflict_keys[@]}" -gt 0 ]; then
   fi
 fi
 
-# Handles all three cases: the KEY=value line is missing entirely (.env
-# predates that var being added to .env.example — sed can't fix what isn't
-# there, so this appends it), present but blank, or already set.
 set_if_blank() {
   local key="$1" value="$2"
   if grep -qE "^${key}=.*[^[:space:]]" .env; then
@@ -172,10 +131,7 @@ set_if_blank() {
 }
 
 echo "==> filling in missing secrets"
-# Random values — safe to re-run, never overwrites what's already set.
+# see docs/operations.md#secrets
-# OMNIROUTE_STORAGE_ENCRYPTION_KEY especially: never change it after first
-# run, existing encrypted data becomes unreadable if you do (same caveat as
-# LiteLLM's old LITELLM_SALT_KEY).
 set_if_blank OMNIROUTE_INITIAL_PASSWORD "$(openssl rand -hex 16)"
 set_if_blank OMNIROUTE_JWT_SECRET "$(openssl rand -base64 48)"
 set_if_blank OMNIROUTE_API_KEY_SECRET "$(openssl rand -hex 32)"
@@ -183,12 +139,9 @@ set_if_blank OMNIROUTE_STORAGE_ENCRYPTION_KEY "$(openssl rand -hex 32)"
 set_if_blank OMNIROUTE_MACHINE_ID_SALT "$(openssl rand -hex 16)"
 set_if_blank OMNIROUTE_CLI_SALT "$(openssl rand -hex 16)"
 set_if_blank OMNIROUTE_WS_BRIDGE_SECRET "$(openssl rand -hex 32)"
-set_if_blank NEO4J_PASSWORD "$(openssl rand -hex 16)"
 
 echo "==> resolving SEARXNG_LAN_IP"
-# search.home is a LAN mDNS/local-DNS name — resolvable from this host, just
+# see docs/gateway.md#web-search
-# not from inside the omniroute container (see docs/research/litellm-searxng-search.md,
-# still the relevant background even though omniroute replaced litellm — see issue #31).
 searxng_ip="$(getent hosts search.home 2>/dev/null | awk '{print $1}' | head -1)"
 if [ -n "$searxng_ip" ]; then
   set_if_blank SEARXNG_LAN_IP "$searxng_ip"
@@ -196,18 +149,8 @@ else
   echo "SEARXNG_LAN_IP: couldn't resolve search.home from this host, set it manually if still blank."
 fi
 
-echo "==> resolving ComfyUI host UID"
-# yurisasc/comfyui-rocm7.1 wants these as env vars, not just group_add in
-# compose — resolve from this host, same pattern as SEARXNG_LAN_IP.
-set_if_blank COMFYUI_PUID "$(id -u)"
-set_if_blank COMFYUI_PGID "$(id -g)"
-
 echo "==> resolving host video/render GIDs (shared by every GPU service)"
-# Numeric GIDs, not names, in docker-compose.yml's group_add: — Docker
+# see docs/architecture.md#gpu-access
-# resolves a *named* group_add entry against the container's own /etc/group,
-# not the host's, and fails unpredictably (worse with multiple GPU services
-# starting concurrently and racing on the same lookup) — see
-# docs/research/rocm-gpu-pin-and-render-group.md and issue #5.
 video_gid="$(getent group video 2>/dev/null | cut -d: -f3)"
 render_gid="$(getent group render 2>/dev/null | cut -d: -f3)"
 if [ -n "$video_gid" ]; then
@@ -232,21 +175,10 @@ docker compose build --pull
 
 echo "==> ensuring models are downloaded (skips already-present files)"
 docker compose --profile tools run --rm downloader
-docker compose --profile tools run --rm downloader-fast
-docker compose --profile tools run --rm downloader-classifier
-docker compose --profile tools run --rm downloader-comfyui
 
 echo "==> bringing up omniroute"
 docker compose up -d --wait omniroute
 
-# ponytail: no scripted key-minting yet, unlike the old LiteLLM /key/generate
-# flow — omniroute's POST /api/keys needs a dashboard login session
-# (ManagementSessionAuth), not a static bearer key, and that flow hasn't
-# been verified against a live instance (see issue #37). No in-stack
-# workload needs a key right now (nothing left calls the gateway besides
-# coding CLIs, which mint their own by hand per docs/proxy-key-onboarding.md)
-# — revisit this script once that flow is automatable.
-
 echo "==> recreating changed services"
 docker compose up -d --remove-orphans
 

speed-bench/Dockerfile

@@ -0,0 +1,9 @@
+# see docs/operations.md#speed-bench
+FROM python:3.12-slim
+ARG LLAMA_CPP_REF=0ec4027a2a0183de87c007069adddc06e665f7de
+ADD https://raw.githubusercontent.com/ggml-org/llama.cpp/${LLAMA_CPP_REF}/tools/server/bench/speed-bench/requirements.txt \
+    https://raw.githubusercontent.com/ggml-org/llama.cpp/${LLAMA_CPP_REF}/tools/server/bench/speed-bench/speed_bench.py \
+    https://raw.githubusercontent.com/ggml-org/llama.cpp/${LLAMA_CPP_REF}/tools/server/bench/speed-bench/speed_bench_compare.py \
+    /app/
+RUN pip install --no-cache-dir -r /app/requirements.txt
+WORKDIR /results