Merge pull request 'docs(research): add qwen3.8-27b decode speed analysis' (#68) from docs/decode-speed-research into main

Reviewed-on: #68

.env.example

@@ -5,10 +5,11 @@ LLAMA_MODEL_FILE=Qwen3.8-27B-UD-Q4_K_XL.gguf
 LLAMA_GPU_LAYERS=999
 LLAMA_CTX_SIZE=262144
 LLAMA_PARALLEL=2
-LLAMA_CACHE_RAM=24576
+LLAMA_CACHE_RAM=8192
+LLAMA_KV_CACHE_TYPE=q8_0
 
-LLAMA_DRAFT_MODEL_FILE=Qwen3.8-27B-DFlash2-Q4_K_M.gguf
+# Blank = speculative decoding off; see docs/architecture.md#llama-server
-LLAMA_SPEC_N_MAX=7
+LLAMA_SPEC_ARGS=--spec-type draft-dflash -md /models/Qwen3.8-27B-DFlash2-Q8_0.gguf --spec-draft-n-max 3 --spec-draft-p-min 0.5
 
 # --- SearXNG web search --- see docs/gateway.md#web-search
 # Auto-filled by update.sh — leave blank.

.gitignore

@@ -5,4 +5,6 @@
 data/
 .leankg/
 .cache/
-.qwen/tmp
+.qwen/tmp
+# speed-bench output, see docs/operations.md#speed-bench
+bench-results/

.serena/.gitignore

@@ -0,0 +1,2 @@
+/cache
+/project.local.yml

.serena/project.yml

@@ -0,0 +1,187 @@
+# the name by which the project can be referenced within Serena/when chatting with the LLM.
+project_name: "LLM-Server"
+
+# list of language servers to start when using the LSP backend; choose from:
+#   ada                    al                     angular                ansible                astro
+#   bash                   bsl                    clojure                cpp                    cpp_ccls
+#   crystal                csharp                 csharp_omnisharp       cue                    dart
+#   deno                   elixir                 elm                    erlang                 fortran
+#   fsharp                 gdscript               gleam                  go                     groovy
+#   haskell                haxe                   hlsl                   html                   java
+#   json                   julia                  julia_fatou            kotlin                 latex
+#   lean4                  lua                    lua_emmylua            luau                   markdown
+#   matlab                 msl                    nextflow               nix                    ocaml
+#   pascal                 perl                   php                    php_devsense           php_phpactor
+#   php_phpantom           powershell             python                 python_basedpyright    python_jedi
+#   python_pyrefly         python_ty              qml                    r                      rego
+#   ruby                   ruby_solargraph        rust                   scala                  scss
+#   solidity               svelte                 swift                  systemverilog          terraform
+#   toml                   typescript             typescript_tsgo        typescript_vts         vue
+#   vbnet                  wolfram                yaml                   zig
+#   (This list may be outdated; generated with scripts/print_language_list.py;
+#   For the current list, see values of the LanguageServerId enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
+# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Astro projects, use astro (subsumes typescript/javascript for .astro projects; requires npm)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+#   - External Python adapters may add further registered IDs; install the adapter package first
+#     and then use its ID here, for example: example
+# Special requirements:
+#   Some language servers require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple language servers, the first language server that supports a given file will be used for that file.
+# The first language server is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+language_servers:
+- bash
+
+# the encoding used by text files in the project
+# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
+encoding: "utf-8"
+
+# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
+# the command runs in the project root directory and is only executed if the project is trusted
+# (see trusted_project_path_patterns in the global configuration).
+# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
+# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
+# backstop for non-terminating commands; on expiry the process is killed and activation continues.
+# example: activation_command: "npx nx run-many -t build"
+activation_command:
+
+# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
+# must be a positive number.
+activation_command_timeout: 180.0
+
+# line ending convention to use when writing source files.
+# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
+# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
+line_ending:
+
+# The language backend to use for this project.
+# If not set, the global setting from serena_config.yml is used.
+# Valid values: LSP, JetBrains
+# Note: the backend is fixed at startup. If a project with a different backend
+# is activated post-init, an error will be returned.
+language_backend:
+
+# The interface through which the agent (LLM) accesses Serena's functionality (overrides the global setting).
+# Valid values: tools, REPL (see the global configuration for details); leave empty to use the global setting.
+# Note: the interface is fixed at startup. If a project is activated post-init, its setting is not applied.
+agent_interface:
+
+# whether to use project's .gitignore files to ignore files
+ignore_all_files_in_gitignore: true
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
+ls_specific_settings: {}
+
+# list of workspace folder paths (LSP backend only).
+# These folders will be used to build up Serena's symbol index.
+# Paths must be within the project root and should thus be relative to the project root.
+# Furthermore, the paths should not be filtered by ignore settings.
+# Default setting: The entire project root folder (".") is considered.
+# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
+#   ls_workspace_folders:
+#     - "./subproject1"
+#     - "./subproject2"
+ls_workspace_folders:
+- "."
+
+# list of additional workspace folder paths for cross-package reference support.
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries, but these folders are not indexed by Serena,
+# i.e. the respective symbols will not be found using Serena's symbol search tools.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+ls_additional_workspace_folders: []
+
+# list of additional paths to ignore in this project.
+# Same syntax as gitignore, so you can use * and **.
+# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
+# Example:
+#   ignored_paths:
+#     - "examples/**"
+#     - ".worktrees/**"
+#     - "**/bin/**"
+#     - "**/obj/**"
+# Note: global ignored_paths from serena_config.yml are also applied additively.
+ignored_paths: []
+
+# whether the project is in read-only mode
+# If set to true, all editing tools will be disabled and attempts to use them will result in an error
+# Added on 2025-04-18
+read_only: false
+
+# list of tool names to exclude.
+# This extends the existing exclusions (e.g. from the global configuration)
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+excluded_tools: []
+
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+fixed_tools: []
+
+# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.find_symbol") to exclude from the REPL.
+# This extends the existing exclusions (e.g. from the global configuration).
+excluded_apis: []
+
+# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.get_diagnostics_for_symbol") to include in the REPL
+# that would otherwise be disabled (particularly optional methods, which are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+included_apis: []
+
+# list of mode names that are to be activated by default, overriding the setting in the global configuration.
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
+# for this project.
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+default_modes:
+
+# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+added_modes:
+
+# initial prompt for the project. It will always be given to the LLM upon activating the project
+# (contrary to the memories, which are loaded on demand).
+initial_prompt: ""
+
+# time budget (seconds) per tool call for the retrieval of additional symbol information
+# such as docstrings or parameter information.
+# This overrides the corresponding setting in the global configuration; see the documentation there.
+# If null or missing, use the setting from the global configuration.
+symbol_info_budget:
+
+# list of regex patterns which, when matched, mark a memory entry as read‑only.
+# Extends the list from the global configuration, merging the two lists.
+read_only_memory_patterns: []
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []

README.md

@@ -22,7 +22,7 @@ It creates and syncs `.env`, generates secrets, downloads the models, and brings
 | [`docs/architecture.md`](docs/architecture.md) | Each service, why it exists, and why it's configured the way it is |
 | [`docs/gateway.md`](docs/gateway.md) | OmniRoute: dashboard, provider connections, API keys, timeouts, web search |
 | [`docs/network-access.md`](docs/network-access.md) | Who can reach which port, and why |
-| [`docs/operations.md`](docs/operations.md) | What `update.sh` does: env sync, secrets, downloads |
+| [`docs/operations.md`](docs/operations.md) | What `update.sh` does: env sync, secrets, downloads; speed bench |
 | [`docs/coding-cli-setup/`](docs/coding-cli-setup/index.md) | Pointing Claude Code, Kimi CLI, OpenCode and Qwen Code at the gateway |
 | [`docs/research/`](docs/research/README.md) | Dated research and decision records, each marked current or superseded |
 | [`docs/server-planing.md`](docs/server-planing.md) | Background research: a 4× GPU rack build. Not an active plan |

docker-compose.yml

@@ -25,12 +25,11 @@ services:
       --parallel ${LLAMA_PARALLEL:-2}
       --kv-unified
       --flash-attn on
-      --cache-type-k q8_0
+      --cache-type-k ${LLAMA_KV_CACHE_TYPE:-q8_0}
-      --cache-type-v q8_0
+      --cache-type-v ${LLAMA_KV_CACHE_TYPE:-q8_0}
-      --cache-ram ${LLAMA_CACHE_RAM:-24576}
+      --cache-ram ${LLAMA_CACHE_RAM:-8192}
-      --spec-type draft-dflash
+      --no-cache-idle-slots
-      -md /models/${LLAMA_DRAFT_MODEL_FILE:-Qwen3.8-27B-DFlash2-Q4_K_M.gguf}
+      ${LLAMA_SPEC_ARGS:-}
-      --spec-draft-n-max ${LLAMA_SPEC_N_MAX:-7}
       --jinja
     # see docs/architecture.md#llama-server (flags), docs/network-access.md#backends (no host port)
     expose:
@@ -38,34 +37,34 @@ services:
     restart: unless-stopped
     networks: [ai-stack]
 
-  # see docs/operations.md#model-downloads (also covers downloader-classifier)
+  # see docs/operations.md#model-downloads; the list of files is models.yml
   downloader:
-    image: curlimages/curl:latest
+    image: alpine:3
     profiles: ["tools"]
     user: root
     volumes:
       - models:/models
-    entrypoint: ["sh", "-c"]
+      - ./models.yml:/models.yml:ro
+    entrypoint: ["sh", "-euo", "pipefail", "-c"]
     command:
-      - >
+      - |
-        test -f /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} &&
+        apk add --no-cache -q curl yq-go
-        echo "already downloaded, skipping" ||
+        yq -r '.models[] | (.file // "") + " " + (.url // "")' /models.yml | while read -r file url; do
-        curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
+          if [ -z "$$url" ]; then echo "models.yml: entry '$$file' needs both file and url" >&2; exit 1; fi
-        https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
+          if [ -f "/models/$$file" ]; then echo "$$file: already downloaded, skipping"; continue; fi
+          curl -L --fail -o "/models/$$file.part" "$$url"
+          mv "/models/$$file.part" "/models/$$file"
+        done
 
-  downloader-draft:
+  # see docs/operations.md#speed-bench
-    image: curlimages/curl:latest
+  speed-bench:
     profiles: ["tools"]
-    user: root
+    build: ./speed-bench
+    entrypoint: ["python", "/app/speed_bench.py", "--url", "http://llama-server:8080"]
     volumes:
-      - models:/models
+      - hf-cache:/root/.cache/huggingface
-    entrypoint: ["sh", "-c"]
+      - ./bench-results:/results
-    command:
+    networks: [ai-stack]
-      - >
-        test -f /models/${LLAMA_DRAFT_MODEL_FILE:-Qwen3.8-27B-DFlash2-Q4_K_M.gguf} &&
-        echo "already downloaded, skipping" ||
-        curl -L --fail --create-dirs -o /models/${LLAMA_DRAFT_MODEL_FILE:-Qwen3.8-27B-DFlash2-Q4_K_M.gguf}
-        https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/${LLAMA_DRAFT_MODEL_FILE:-Qwen3.8-27B-DFlash2-Q4_K_M.gguf}
 
   # see docs/architecture.md#omniroute; runtime config lives in its dashboard, see docs/gateway.md
   omniroute:
@@ -127,3 +126,4 @@ volumes:
   models:
   omniroute-data:
   qdrant-data:
+  hf-cache:

docs/architecture.md

@@ -21,14 +21,15 @@ The main model's backend. It is always running. Lazytainer used to idle-stop it,
 
 - **Model file** (`LLAMA_MODEL_FILE`): the quant choice and its VRAM budget are in [`research/qwen3.8-27b-quant.md`](research/qwen3.8-27b-quant.md).
 - **`LLAMA_GPU_LAYERS`**: the default puts every layer on the GPU. The model is dense, not Mixture-of-Experts, so this is the only RAM-offload knob. `--n-cpu-moe`, `--cpu-moe` and `--override-tensor "exps"` do nothing here. There is no SSD tier to enable either. llama.cpp mmaps the model file, so if GPU and RAM ever can't hold it, the OS pages the rest in from disk as a slow, implicit last resort. Explicit tiered offload is an [open llama.cpp request](https://github.com/ggml-org/llama.cpp/discussions/12507).
-- **`LLAMA_CTX_SIZE`**: set to the model's own maximum (`max_position_embeddings`). It only fits beside the weights because the KV cache is quantized (`--cache-type-k/v`). An fp16 KV cache at that size would not fit the card. The numbers are in the quant research doc.
+- **`LLAMA_CTX_SIZE`**: set to the model's own maximum (`max_position_embeddings`). It only fits beside the weights because the KV cache is quantized (`LLAMA_KV_CACHE_TYPE`, used for both `--cache-type-k` and `--cache-type-v`). An fp16 KV cache at that size would not fit the card. The numbers are in the quant research doc.
 - **`LLAMA_PARALLEL`** is the backend's number of **slots**. It's kept low on purpose: more slots meant more contended prefill, which pushed requests past the gateway's timeouts ([gateway → Timeouts](gateway.md#timeouts)). Requests beyond the slot count wait in llama.cpp's own queue, which has no timeout. So the timeout that matters is the gateway's per-connection one.
 - **Per-slot context = `LLAMA_CTX_SIZE / LLAMA_PARALLEL`.** llama.cpp divides the context across slots, and each request gets one slot's share. Every coding CLI's context-window setting must use this number, not the raw context size. Real sessions already fill a large share of it, so check that the per-slot figure stays comfortably above observed usage before lowering either value.
 - **No `--cache-reuse`:** this model's context doesn't support it. llama-server logs "cache_reuse is not supported by this context, it will be disabled" at startup, so the flag was dropped. Re-prefill avoidance comes from `--cache-ram` (below) and OmniRoute's prompt-cache affinity (a dashboard setting, on by default), which keeps a conversation on the same slot. The earlier fix for the gateway's non-ping SSE timeout ([`research/omniroute-non-ping-sse-stream-timeout.md`](research/omniroute-non-ping-sse-stream-timeout.md)) assumed `--cache-reuse` worked, so long re-prefills can still hit that timeout.
-- **`LLAMA_CACHE_RAM`** (`--cache-ram`, MiB) is the host-RAM budget for saved prompt-cache entries. When a slot is reused, llama.cpp parks the old prompt's state in RAM so it can be restored instead of re-prefilled. The llama.cpp default is too small for ~40k-token agent prompts (entries are ~2 GiB each), so entries got evicted and requests re-prefilled in full. Raising it is the only place spare DDR RAM helps; it does not speed up decode. Keep it well below the host's free RAM. Evidence and the A/B method: [`research/qwen3.8-27b-throughput-tuning.md`](research/qwen3.8-27b-throughput-tuning.md).
+- **Testing for token corruption** (typos, broken tool-call keys): flip one key at a time in the server's `.env`, then run `docker compose up -d llama-server`. Blank `LLAMA_SPEC_ARGS` rules speculative decoding in or out, `LLAMA_KV_CACHE_TYPE=f16` does the same for the quantized KV cache (lower `LLAMA_CTX_SIZE` so it fits), and `LLAMA_CACHE_RAM=0` turns off the RAM prompt cache. A drafter's quant can't cause wrong tokens: the target model verifies every draft token, so a worse drafter only lowers acceptance.
+- **`LLAMA_CACHE_RAM`** (`--cache-ram`, MiB) is the host-RAM budget for saved prompt-cache entries. When a slot is reused, llama.cpp parks the old prompt's state in RAM so it can be restored instead of re-prefilled. The llama.cpp default is too small for ~40k-token agent prompts (entries are ~2 GiB each), so entries got evicted and requests re-prefilled in full. Raising it is the only place spare DDR RAM helps; it does not speed up decode. Keep it well below the host's free RAM. `--no-cache-idle-slots` stays on while [llama.cpp#27148](https://github.com/ggml-org/llama.cpp/issues/27148) is open: with the RAM cache on, an LRU-picked slot can keep an unrelated conversation's state, which showed up as misspelled output. With `--kv-unified`, turning off idle-slot caching closes that path and keeps the RAM cache ([diagnosis](research/inference-engine-spec-decoding-multi-gpu.md)). Evidence and the A/B method: [`research/qwen3.8-27b-throughput-tuning.md`](research/qwen3.8-27b-throughput-tuning.md).
 - **`--jinja`** enables the chat template that tool calling depends on. Without it, tool-use requests fail outright.
 - **Tool-calling reliability** of this model family on llama.cpp has open upstream parser bugs: [`research/qwen3.8-27b-tool-calling.md`](research/qwen3.8-27b-tool-calling.md).
-- **Speculative decoding (DFlash2):** `--spec-type draft-dflash` with a separate drafter GGUF (`LLAMA_DRAFT_MODEL_FILE`), fetched by `downloader-draft`. It replaced MTP (draft heads inside the target GGUF). `LLAMA_SPEC_N_MAX` is the draft length. Upstream: [llama.cpp#27342](https://github.com/ggml-org/llama.cpp/pull/27342). Draft acceptance can collapse at high slot counts ([llama.cpp#27117](https://github.com/ggml-org/llama.cpp/issues/27117)), another reason `LLAMA_PARALLEL` stays low.
+- **Speculative decoding (DFlash2):** all its flags live in one key, `LLAMA_SPEC_ARGS`: `--spec-type draft-dflash`, the drafter GGUF (`-md`, fetched by the downloader from [`models.yml`](../models.yml)) the draft length (`--spec-draft-n-max`), and `--spec-draft-p-min`, which ends a draft early once DFlash2's own confidence in the next token drops below the threshold. Long thinking output at the recommended sampling temperature gets low acceptance, so short confident drafts waste less verify work than long ones ([`common/speculative.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/common/speculative.cpp), DFlash2 lattice walk). One key means a single `.env` edit swaps the drafter or turns speculation off (leave it blank) for an A/B test. A blank server value differs from `.env.example`, so `update.sh` will ask about it on every run until it's restored ([operations → Env sync](operations.md#env-sync)). It replaced MTP (draft heads inside the target GGUF). Upstream: [llama.cpp#27342](https://github.com/ggml-org/llama.cpp/pull/27342). Draft acceptance can collapse at high slot counts ([llama.cpp#27117](https://github.com/ggml-org/llama.cpp/issues/27117)), another reason `LLAMA_PARALLEL` stays low.
 - **No classifier backend:** the separate `qwen-classifier` was removed to free VRAM for the drafter. Anything that needs a small "fast" model must use the main model.
 - **No host port**: see [network access → Backends](network-access.md#backends).
 

docs/operations.md

@@ -1,6 +1,6 @@
 # Operations
 
-> **Source of truth:** [`scripts/update.sh`](../scripts/update.sh) and [`.env.example`](../.env.example). This page explains *what the script does and why*; the steps and defaults live in the source. If they disagree, the source wins.
+> **Source of truth:** [`scripts/update.sh`](../scripts/update.sh), [`scripts/redownload-model.sh`](../scripts/redownload-model.sh), [`.env.example`](../.env.example), [`models.yml`](../models.yml), and the `speed-bench` service in [`docker-compose.yml`](../docker-compose.yml) with its [`speed-bench/Dockerfile`](../speed-bench/Dockerfile) and [`scripts/bench.sh`](../scripts/bench.sh). This page explains *what the script does and why*; the steps and defaults live in the source. If they disagree, the source wins.
 
 The stack runs on the **server**. After any change to the repo, run `./scripts/update.sh` there: it is the one command that brings the running stack in sync. It's safe to re-run at any time, and it only recreates what changed.
 
@@ -39,4 +39,35 @@ The conflict screen uses [gum](https://github.com/charmbracelet/gum), a single s
 
 ## Model downloads
 
-Each model file has its own one-off downloader service in the `tools` compose profile. `update.sh` runs them every time. A `test -f` guard skips files that already exist, so re-runs don't re-download. Files live in the named `models` volume, not a host bind-mount. The downloaders run as root because the named volume is created root-owned. There's one service per model, so the small model's job never re-checks the large one.
+One `downloader` service in the `tools` compose profile fetches every file listed in [`models.yml`](../models.yml) (`file` + `url` per entry). `update.sh` runs it every time. Files that already exist are skipped, so re-runs don't re-download. To add a model, add an entry there; to load it, point the matching `LLAMA_*_FILE` key in `.env` at the same `file` name. The two are not linked, so a typo means llama-server can't find the file.
+
+- **Partial downloads** go to `<file>.part` and are only renamed once complete. Otherwise an interrupted download would leave a truncated file that the skip check treats as done.
+- **Files live in the named `models` volume**, not a host bind-mount. The downloader runs as root because the named volume is created root-owned.
+- **It needs internet on each run**: it installs `curl` and `yq` from Alpine's package mirror before reading the list.
+
+### Forcing a model re-download
+
+The skip check means `update.sh` never picks up an upstream change to a file that's already on disk (for example, a re-uploaded GGUF with the same name). [`scripts/redownload-model.sh`](../scripts/redownload-model.sh) forces it: with no argument it refreshes `LLAMA_MODEL_FILE` from `.env`, or pass a `file` name from `models.yml` (for example the drafter). It reuses the compose downloader, so the URL still lives only in `models.yml`.
+
+- The old file is renamed to `<file>.old` and deleted only after the new download succeeds. If the download fails or is interrupted, the old file is put back. This needs room for both copies in the `models` volume at once.
+- It ends by restarting `llama-server`, because the running process keeps the old file mapped until it restarts.
+
+## Speed bench
+
+`speed-bench` runs llama.cpp's [SPEED-Bench client](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/bench/speed-bench/README.md) against the running `llama-server`. It measures decode speed (`avg_pred_t/s`), latency and draft acceptance per category. Use it to compare drafters and their flags with numbers instead of gut feel. It's in the `tools` profile, so `update.sh` never builds or runs it. The first run builds the image.
+
+- **The scripts are pinned** to one llama.cpp commit (`LLAMA_CPP_REF` in the Dockerfile). The server image is not pinned, so bump the ref if the client stops matching the server's response format.
+- **It reaches the backend over the compose network**, so llama-server needs no host port. The dataset (`nvidia/SPEED-Bench`, not gated) downloads on first run and is cached in the `hf-cache` volume. Results land in `bench-results/` (gitignored).
+- **It measures speed, not correctness.** The output text isn't saved, so it won't catch a drafter that corrupts tokens.
+
+Run it on the server with `./scripts/bench.sh`. It wraps the compose calls and creates `bench-results/` first, so Docker doesn't create it root-owned. Extra arguments go to the SPEED-Bench client. Pass `--concurrency` equal to `LLAMA_PARALLEL` for throughput, or leave it at 1 for the single-session speed one CLI sees:
+
+```sh
+./scripts/bench.sh run spec --category coding --limit 8
+```
+
+To get a speedup figure, also run a **baseline** without speculative decoding. Temporarily remove the `--spec-*` and `-md` lines from the llama-server command, run `docker compose up -d llama-server`, then run `./scripts/bench.sh run baseline` with the same arguments. Keep `--bench`, `--category`, `--osl` and `--limit` identical. Restore the lines with `git checkout docker-compose.yml` and `docker compose up -d llama-server`, then compare:
+
+```sh
+./scripts/bench.sh compare baseline spec
+```

docs/research/README.md

@@ -11,6 +11,7 @@ Each file here is a **dated snapshot**: research or a decision record as it stoo
 | [colibri-ai-stack-evaluation.md](colibri-ai-stack-evaluation.md) | Current | Rejected option ("watch, don't integrate"). It lists ComfyUI, which has since been removed |
 | [connecting-cloud-providers.md](connecting-cloud-providers.md) | Superseded by the OmniRoute migration | Written for LiteLLM. It mentions Open WebUI, which has been removed |
 | [dashscope-websearch-selfhost-options.md](dashscope-websearch-selfhost-options.md) | Current | Why DashScope web search isn't emulated |
+| [dual-r9700-speed-and-offload.md](dual-r9700-speed-and-offload.md) | Current | Second R9700: split modes, 27B Q8_K_XL vs Flash-Next, RAM/SSD offload, Strata. Proposals only, nothing applied |
 | [fast-model-choice.md](fast-model-choice.md) | Current (drift) | Classifier model choice. Its addendum records the service name (`qwen-classifier`) and quant drift |
 | [langchain-pgvector-vs-litellm-pgvector.md](langchain-pgvector-vs-litellm-pgvector.md) | Superseded by the OmniRoute migration | LiteLLM-era RAG comparison |
 | [lazytainer-omniroute-idle-stop.md](lazytainer-omniroute-idle-stop.md) | Superseded: Lazytainer and `switch-model.sh` removed | Root cause of why Lazytainer's idle-stop never fired behind the gateway, which is the reason it was removed |
@@ -27,6 +28,8 @@ Each file here is a **dated snapshot**: research or a decision record as it stoo
 | [proxy-request-priority.md](proxy-request-priority.md) | Superseded by the OmniRoute migration | Priority tiers via LiteLLM's scheduler, never built on OmniRoute |
 | [proxy-shadow-pricing.md](proxy-shadow-pricing.md) | Superseded by the OmniRoute migration | Shadow cost was wired into LiteLLM config, and nothing implements it now |
 | [proxy-tool-choice.md](proxy-tool-choice.md) | Superseded by the OmniRoute migration | Chose LiteLLM. It mentions Open WebUI, which has been removed |
+| [inference-engine-spec-decoding-multi-gpu.md](inference-engine-spec-decoding-multi-gpu.md) | Current | llama.cpp vs. vLLM vs. SGLang on R9700: drafters, multi-GPU, licenses, `--cache-ram` misspelling bug. Not benchmarked |
+| [qwen3.8-27b-decode-speed.md](qwen3.8-27b-decode-speed.md) | Current | Decode tokens/s: why DFlash2 acceptance and speed fall with depth (draft length is 5, not 7), n_max/p_min, KV and slot costs, bandwidth ceiling, ordered experiment plan. Builds on the throughput-tuning note. Proposals only, nothing applied |
 | [qwen3.8-27b-quant.md](qwen3.8-27b-quant.md) | Current | Main model quant and VRAM budget |
 | [qwen3.8-27b-throughput-tuning.md](qwen3.8-27b-throughput-tuning.md) | Current | Tokens/s tuning from the server log, DDR RAM as a cache tier, comparison with five outside R9700 configs. Proposals only, nothing applied |
 | [qwen3.8-27b-tool-calling.md](qwen3.8-27b-tool-calling.md) | Current | Tool-calling parser risk for the main model |

docs/research/dual-r9700-speed-and-offload.md

@@ -0,0 +1,240 @@
+# Research: fastest inference on 2x R9700 (27B Q8_K_XL vs Flash-Next, multi-GPU modes, RAM/SSD offload, Strata)
+
+**Date:** 2026-10-29
+
+**Question:** A second Radeon AI PRO R9700 arrives in about a week. What gives the fastest decode (and prefill) for agentic coding on this box: `Qwen3.8-27B-UD-Q8_K_XL`, `Qwen3.8-Flash-Next-UD-Q4_K_XL`, or something else? Which llama.cpp multi-GPU mode helps, does RAM/SSD offload work with 32 GB of RAM, and is Strata worth switching to?
+
+**Scope note:** research only. Nothing in `docker-compose.yml` or `.env.example` was changed, so nothing needs deploying.
+
+**Hardware assumed:** Ryzen 9 5950X, 32 GB DDR4 dual channel (reported ~94 % used, swap in use), ASRock B550M Pro4, 1 TB NVMe, Ubuntu 24.04, kernel 6.8, 1x R9700 now and 2x soon.
+
+## TL;DR
+
+| # | Finding | Confidence |
+|---|---|---|
+| 1 | **The second card does not make the dense 27B decode faster with `-sm layer`.** Layer split runs the cards one after the other, so one token still reads every weight at one card's bandwidth. It only adds VRAM. | High (llama.cpp docs) |
+| 2 | **`UD-Q8_K_XL` is a quality move, not a speed move.** It reads ~30.1 GB per token against ~16.8 GB for today's `UD-Q4_K_XL`. Estimated decode is ~15 t/s without speculation and ~25 t/s with DFlash2, against ~46 t/s measured today. | Medium (estimate, math below) |
+| 3 | **`-sm tensor` is the only mode that can speed up one stream, and on ROCm it is immature.** The one measured 2-card ROCm result for this exact model is 1.31x one card (2x 7900 XTX). DFlash2 crashes with `-sm tensor` (open bug), and MTP plus `-sm tensor` has lockup reports. Your second slot is PCIe 3.0 x4 behind the chipset, which hurts the per-layer all-reduces. | Medium |
+| 4 | **Best use of the second card for 2 parallel agents: two independent single-GPU `llama-server` replicas**, one per card, each `--parallel 1`. Each agent keeps today's ~46 t/s, the decode drop when the other slot prefills goes away, and each slot gets the full 262144 tokens instead of 131072. Needs a second backend connection in OmniRoute with per-conversation stickiness. | Medium-high |
+| 5 | **Flash-Next UD-Q4_K_XL (111.3 GB) does not fit.** Even with the 28.8 GB n-gram table left on SSD, 82.5 GB of weights remain for 64 GB of VRAM. The ~20-24 GB of experts left over would come from NVMe, because RAM is full. Estimated decode is then ~6-12 t/s. Only the all-in-VRAM quants (`UD-Q2_K_XL`, `UD-IQ3_XXS`) are realistic, at a large quality cost (mean KLD 0.22 / 0.17 vs 0.047). | Medium |
+| 6 | **RAM is the binding constraint, and compose itself is a likely cause of the swap.** `--cache-ram` is set to 24576 MiB (24 GiB) on a 32 GB host that also runs OmniRoute with `mem_limit: 10g`. | High for the config, inferred for the swap |
+| 7 | **Strata:** a real, fast-moving MIT engine built around Flash-Next expert tiering. It has R9700 HIP builds and a layer split for 2 cards, but its design wants all experts in RAM (~88 GB for UD-Q4_K_XL), it does not run the dense 27B at all, and its numbers are self-reported on hosts with 47-503 GB of RAM. Not worth it on 32 GB. Revisit after a RAM upgrade (board max 128 GB). | Medium |
+
+## 1. Platform facts that drive everything
+
+- **R9700:** 32 GB GDDR6, 256-bit, 20 Gbps, **640 GB/s**, PCIe 5.0 x16, 300 W ([Puget Systems listing](https://www.pugetsystems.com/parts/Video-Card/AMD-Radeon-AI-PRO-R9700-32GB-16908/), [Tom's Hardware launch](https://www.tomshardware.com/pc-components/gpus/amd-launches-radeon-ai-pro-r9700-to-challenge-nvidias-ai-market-dominance)). AMD's own product page timed out on fetch, so this is from third-party listings.
+- **Effective bandwidth, calibrated from this repo's data:** no-spec decode for the 27B Q4 on one R9700 is 25-30 t/s across three outside sources ([`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md)). 16.8 GB/token x ~26.4 t/s ≈ **~440 GB/s effective (~69 % of peak)**. All estimates below use 420-450 GB/s.
+- **ASRock B550M Pro4 slots**, from the [ASRock manual (PDF)](https://download.asrock.com/Manual/B550M%20Pro4.pdf) spec page with a Matisse/Vermeer CPU: "2 x PCI Express x16 Slots (PCIE1: Gen4x16 mode; PCIE3: Gen3 x4 mode)", plus one PCIe 3.0 x1 slot (PCIE2). M2_1 is "up to Gen4x4", M2_2 "up to Gen3 x2", and M2_2 shares lanes with SATA3_5_6. Memory is dual channel, max 128 GB. So **the second R9700 runs at PCIe 3.0 x4 (~3.9 GB/s per direction)**.
+- **PCIE3 is behind the chipset.** The manual does not say so explicitly. It follows from the AM4 lane budget: the CPU gives x16 to the GPU, x4 to the CPU-attached M.2, and x4 as the chipset uplink. B550's own lanes and uplink are PCIe 3.0 ([ComputerBase B550 test](https://www.computerbase.de/2020-06/amd-b550-chipsatz-mainboard-test/), [Guru3D](https://www.guru3d.com/review/amd-ryzen-3-3100-and-3300x-review/page-3/)). Card 2 therefore shares the chipset's PCIe 3.0 x4 uplink with M2_2, SATA, USB and LAN. GPU-to-GPU traffic crosses CPU root complex → chipset → slot. Whether ROCm P2P works across that path was not verified. Assume it falls back to staging through host memory.
+- **DDR4:** dual-channel DDR4-3200 peaks at 2 x 8 B x 3200 MT/s = **51.2 GB/s**. Plan for ~35-45 GB/s effective (estimate).
+- **RAM budget today:** `docker-compose.yml` sets `--cache-ram ${LLAMA_CACHE_RAM:-24576}` (24 GiB) and OmniRoute `mem_limit: 10g` (`OMNIROUTE_MEMORY_MB=8192`). That is 34 GiB of permitted use on a 32 GB host before the OS, qdrant and page cache. `--cache-ram` is a ceiling that fills up over time, which fits the reported 94 % plus swap. `docs/architecture.md` already says to "keep it well below the host's free RAM".
+- **Physical and power, unverified:** two 300 W cards plus a 142 W-PPT 5950X need a PSU of roughly 850 W or more. [`500k-context-…roadmap.md`](500k-context-2-parallel-agents-gpu-roadmap.md) already flagged PSU headroom as the limit. A dual-slot card in the bottom slot of a micro-ATX board can also need a fifth rear bracket in the case. Check both before the card arrives.
+
+## 2. Qwen3.8-27B `UD-Q8_K_XL`
+
+**File facts** (HF tree API plus the GGUF header, parsed from the first 12 MB of each file):
+
+| | UD-Q4_K_XL (today) | UD-Q8_K_XL |
+|---|---|---|
+| File size | 17.56 GB | **31.46 GB** |
+| Tensor types | mixed IQ4/Q4_K-Q6_K, Q8_0 for small tensors | Q8_0 FFN/attention, **BF16** `attn_q/k/v` and `output` |
+| `token_embd` (row lookup, not streamed) | 0.72 GB | 1.35 GB |
+| **Bytes streamed per token** | **~16.8 GB** | **~30.1 GB (1.79x)** |
+
+Sources: [HF tree API](https://huggingface.co/api/models/unsloth/Qwen3.8-27B-GGUF/tree/main), [file page](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF?show_file_info=Qwen3.8-27B-UD-Q8_K_XL.gguf). Both files carry the built-in MTP head (`qwen35.block_count = 65`, `nextn_predict_layers = 1`).
+
+**Architecture** ([unsloth config.json](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/raw/main/config.json)): dense, `Qwen3_5ForConditionalGeneration`, 64 layers, hybrid. 48 Gated DeltaNet (`linear_attention`) layers and 16 `full_attention` layers (`full_attention_interval: 4`), 24 Q heads, 4 KV heads, head_dim 256, 262144 native context. Every weight is read for every token.
+
+**Fit on 2x32 GB.** KV is 16 layers x 2 x 4 heads x 256 x 1.0625 B (q8_0) ≈ 34.8 KB/token, so ~9.1 GB at 262144 tokens. 31.5 GB weights + 1.1 GB DFlash2 Q4_K_M ([z-lab GGUF tree](https://huggingface.co/api/models/z-lab/Qwen3.8-27B-DFlash2-GGUF/tree/main)) + 9.1 GB KV + ~3-5 GB compute buffers per card comes to **~46-50 GB of 64 GB** (estimate). It fits with room. It does not fit on one card.
+
+**Decode estimate** (bandwidth-bound, ~440 GB/s per card):
+
+| Config | Math | Est. decode, no spec | With speculation (x1.7, the ratio this repo measures for DFlash2: 45.9 / ~27) |
+|---|---|---|---|
+| Q4_K_XL, 1 card (today) | 16.8 / 440 = 38 ms | ~26 t/s (measured 25-30) | **45.9 t/s measured** |
+| Q8_K_XL, `-sm layer` | 30.1 / 440 = 68 ms, cards in series | **~15 t/s** | **~25 t/s** |
+| Q8_K_XL, `-sm tensor` at 1.31x | 15 x 1.31 | ~19 t/s | DFlash2 crashes here (see §3). MTP x1.5 gives ~29 t/s if stable |
+| Q4_K_XL, `-sm tensor` at 1.31x | 26 x 1.31 | ~34 t/s | MTP x1.5 gives ~50 t/s if stable |
+
+All of these except the measured row are estimates. Q8_K_XL roughly halves single-stream speed against today. Pick it only if Q4 quality is shown to be the bottleneck on your own tasks.
+
+## 3. llama.cpp multi-GPU on ROCm
+
+Source: [`docs/multi-gpu.md`](https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md) and `common/arg.cpp` on master (fetched 2026-10-29). `--split-mode` takes `none, layer, row, tensor`.
+
+| Mode | What it does | Decode on bandwidth-bound cards | Status |
+|---|---|---|---|
+| `layer` (default) | Pipeline: each GPU owns a contiguous block of layers and their KV | **No single-stream gain.** One token walks GPU0 then GPU1, each reading only its half, so the total time is the same. Prefill can overlap ubatches across GPUs. The docs say: "priority is a fast prefill. Can tolerate slow interconnect speeds" | Mature |
+| `row` | Old row-split of dense matmuls | Docs: "**Deprecated** … Superseded by `tensor`". [PR #19378](https://github.com/ggml-org/llama.cpp/pull/19378): needs "a synchronization after every single operation", only worth it on old slow GPUs | Avoid |
+| `tensor` | Meta-backend tensor parallelism: weights **and** KV split, all-reduce at sync points | Both cards read their half of each layer at the same time, so it is the only mode with a decode win. Docs: "Performance should be good for multiple NVIDIA GPUs using the CUDA backend, no guarantees otherwise" | **EXPERIMENTAL**. Merged in [#19378](https://github.com/ggml-org/llama.cpp/pull/19378) (2026-04-09) |
+
+**Measured `-sm tensor` numbers that are relevant here:**
+
+- **Qwen3.8-27B Q4_K_XL, 2x RX 7900 XTX, ROCm: 47.06 t/s = 1.31x one card.** Per-card DRAM utilization drops from 65.9 % to 43.0 % when split ([#28863](https://github.com/ggml-org/llama.cpp/issues/28863), closed without a fix). This is the best proxy for a 2x R9700 result.
+- The #19378 author on ROCm (RX 6800 + MI50, RX 9060 XT + MI100): "the performance is bad vs. the `-sm layer` baseline". On 2x RTX 4090, the dense Gemma4 31B decoded **1.36x** faster, while pp2048 fell to 0.61x and the MoE model got slower.
+- [#27825](https://github.com/ggml-org/llama.cpp/pull/27825) (merged 2026-09-15) turned on the internal AllReduce for HIP. On an RX 9070 at x16 plus an RX 6800 XT at **PCIe 4.0 x4**, Gemma4 31B Q6_K: tg 23.62 → 24.15 (+2 %), pp +16 %. RCCL is opt-in at build time (`-DGGML_HIP_RCCL=ON`), "disabled because … it was not universally beneficial" (multi-gpu.md). Whether the official `server-rocm` image builds with RCCL was not checked.
+- 2x R9700 user, Gemma4 26B-A4B Q8_0: `-sm tensor` f16 KV 29.7 t/s vs `-sm layer` 19 t/s ([#23567](https://github.com/ggml-org/llama.cpp/issues/23567)). Same card, a different (MoE) model.
+- A 5950X + dual-R9700 user runs "Qwen 3.6 27B Q8 … with tensor split" and reports trouble mixing tensor-split and layer-split processes on ROCm ([#28582](https://github.com/ggml-org/llama.cpp/issues/28582), open).
+
+**Constraints and bugs** that apply to this stack:
+
+- Architecture support: `qwen35` and `qwen4exp` are **not** on the deny list in `llm_arch_supports_sm_tensor` (`src/llama-arch.cpp`, master). `-sm tensor` was re-enabled for qwen4exp in [#28569](https://github.com/ggml-org/llama.cpp/pull/28569).
+- `-sm tensor` needs flash-attn on (`llama-context.cpp`). Quantized KV is supported since [#23792](https://github.com/ggml-org/llama.cpp/pull/23792) (2026-06-01). llama.cpp's `docs/multi-gpu.md` still says it errors, so the doc is stale there. `--fit` is "not supported with `tensor`", so set `--ctx-size` by hand.
+- **DFlash2 + `-sm tensor` crashes:** `GGML_ASSERT(src_ss[0].axis != GGML_BACKEND_SPLIT_AXIS_0)` ([#28777](https://github.com/ggml-org/llama.cpp/issues/28777), open). The ROCm variant ([#27829](https://github.com/ggml-org/llama.cpp/issues/27829)) was closed as stale, and the fix PR [#27858](https://github.com/ggml-org/llama.cpp/pull/27858) is open. Its tester still hit a different assert, and a commenter says "this is already a problem for ROCm". The workaround in #28777 is `-sm layer`.
+- **MTP + `-sm tensor`:** reproducible lockups on CUDA ([#27122](https://github.com/ggml-org/llama.cpp/issues/27122), closed as stale). One ROCm report (2x V620, Q8_K_XL) saw one hard abort in ~67 h.
+- **gfx1201 flash-attn garbage at some context lengths for Qwen3.8-27B** (head_dim 256): immediate EOS in bands, reproduced on **2x R9700, Linux, `--tensor-split 1.7,1`** ([#30884](https://github.com/ggml-org/llama.cpp/issues/30884), open). Another commenter could not reproduce it through `/v1/chat/completions`. Run a needle test on any new config.
+
+**PCIe x4 chipset slot, estimated effects:**
+
+- `-sm layer` decode: one hidden-state hop per token (5120 x 4 B ≈ 20 KB). Negligible. Prefill: ~10 MB per 512-token ubatch hop, ~3 ms at 3.9 GB/s. Small.
+- `-sm tensor` decode: ~2 all-reduces per layer x 64 layers ≈ 128 small reductions per token. These are latency-bound, not size-bound. At an assumed 20-50 µs each through host memory, that is 2.5-6.4 ms per token, 10-25 % of a 25 ms token (estimate). This is the likely reason #28863 lands at 1.31x and not ~1.9x, and an x4-through-chipset link will not do better.
+- `-sm tensor` prefill: ~2 x tokens x 5120 x 2 B per layer. For a 2048-token ubatch that is ~2.7 GB over 64 layers, ~0.7 s at 3.9 GB/s, so prefill is capped at roughly 3000 t/s plus compute (estimate). Expect prefill to get worse than layer split, as in the #19378 tables.
+
+**Flags:**
+
+- `-ts/--tensor-split` sets proportions. With `layer` the default split follows free memory. With `tensor` the default is an even split.
+- `-mg/--main-gpu` applies only to `-sm none` (pick the card) or `row` (intermediate results). It does nothing for `layer`/`tensor`.
+- Draft placement: `--spec-draft-device/-devd` (default: follows `--device`), `-ngld`, `-otd` (`common/arg.cpp`). Under `layer`, the output head sits on the last GPU. Place the drafter on one card and A/B both cards. No source documents which placement is better for DFlash2, which reads hidden states from several target layers (`n_extract=5`) spread over both cards.
+
+## 4. Qwen3.8-Flash-Next `UD-Q4_K_XL`
+
+**Model facts** ([unsloth model card](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF), GGUF header of shard 1, and shard headers 2-4):
+
+- `general.architecture = qwen4exp`, described as "A Preview of the Qwen4 Architecture". **MoE: 125B total, 6B activated, plus a 51B n-gram embedding and a 4B MTP head.** 48 layers, hidden size 2560, layout 12 x (3 x Gated DeltaNet + 1 x Qwen Sparse Attention), each layer followed by an MoE block. **512 experts, 10 routed + 1 shared**, expert FFN dim 640. Native context 262144. License `qwen-community-1.0`. The 27B is Apache 2.0 per [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md).
+- **UD-Q4_K_XL = 111.33 GB** in 4 shards (0.01 + 49.86 + 49.38 + 12.09 GB, [HF tree API](https://huggingface.co/api/models/unsloth/Qwen3.8-Flash-Next-GGUF/tree/main?recursive=true)). It splits into:
+  - `per_layer_token_embd` (n-gram table): **28.80 GB** IQ4_NL, read a few rows per token. The same size in every UD quant.
+  - routed experts `ffn_{gate,up,down}_exps`: **77.02 GB**
+  - everything else (attention/DeltaNet, shared experts, routers, hyper-connections, output head): **~4.8 GB**, mostly Q8_0
+- **Bytes per token:** 4.8 + 77.02 x 10/512 (1.50) ≈ **~6.3 GB**, 2.7x less than the 27B Q4. The all-in-VRAM ceiling is 6.3 / 440 → ~70 t/s. Batch-1 MoE kernels gather scattered experts and reach less of peak, so the estimate is **~40-60 t/s if it fit in VRAM**. For scale, Strata reports 63 (prose) / 87 (code) t/s on one R9700 with this file and its own engine (§6).
+- **Quality** (Qwen card, BF16): SWE-bench Pro 62.5 vs 61.7 for the 27B, SWE-bench Multilingual 81.0 vs 73.8, DeepSWE 58.7 vs 42.2, Toolathlon 73.5 vs 67.1. Unsloth's quant table ([run guide](https://unsloth.ai/docs/models/qwen3.8-next)) gives mean KLD: UD-Q2_K_XL 0.225, UD-IQ3_XXS 0.165, UD-Q3_K_XL 0.107, UD-IQ4_XS 0.084, **UD-Q4_K_XL 0.047**.
+
+**Fit in 64 GB VRAM** (non-PLE = total − 28.8 GB, the PLE left on SSD via `--lazy-mode`, §5):
+
+| Quant | Total | Non-PLE (needs VRAM) | Result on 2x32 GB |
+|---|---|---|---|
+| UD-Q4_K_XL | 111.3 | **82.5** | ~20-24 GB of experts must live off-GPU. **Does not fit** |
+| UD-IQ4_XS | 93.7 | 64.9 | ~6-9 GB off-GPU after KV and buffers |
+| UD-Q3_K_XL | 90.0 | 61.2 | Borderline, a few GB off-GPU |
+| UD-IQ3_XXS | 82.0 | 53.2 | **Fits**, ~10 GB headroom |
+| UD-Q2_K_XL | 78.9 | 50.1 | **Fits**, ~13 GB headroom |
+
+KV is small. Only the 12 QSA layers have one: 12 x 2 KV heads x 256 x 2 (K+V) x 2 B ≈ 24.6 KB/token f16, ~6.4 GB at 262144 tokens, ~3.4 GB at q8_0 (estimate; ignores indexer keys).
+
+**llama.cpp support status:**
+
+- Merged: model support [#27742](https://github.com/ggml-org/llama.cpp/pull/27742) (2026-08-27), MTP [#29761](https://github.com/ggml-org/llama.cpp/pull/29761) (2026-10-01), `-sm tensor` re-enabled [#28569](https://github.com/ggml-org/llama.cpp/pull/28569), many QSA fixes through 2026-10-27 (e.g. [#31243](https://github.com/ggml-org/llama.cpp/pull/31243)).
+- **Open bugs that hit this setup:**
+  - [#29149](https://github.com/ggml-org/llama.cpp/issues/29149): **HIP livelock during load on an R9700**, with both UD-IQ3_XXS and UD-Q4_K_XL. The root-cause comment places it in the mmap upload path; "the non-mmap path stages through pinned buffers instead". Try `--load-mode none`.
+  - [#29811](https://github.com/ggml-org/llama.cpp/issues/29811) and [#30860](https://github.com/ggml-org/llama.cpp/issues/30860): MTP asserts at startup.
+  - [#30887](https://github.com/ggml-org/llama.cpp/issues/30887): the DFlash2 drafter crashes on the first prompt (R9700, ROCm).
+  - [#30815](https://github.com/ggml-org/llama.cpp/pull/30815): QSA chunking, still open.
+
+  Usable, but rough on HIP.
+
+## 5. Offloading to DDR RAM and SSD
+
+**What llama.cpp offers** (master `common/arg.cpp`):
+
+- `-ot/--override-tensor <regex>=<buft>` places matching tensors, e.g. `blk\.[0-9]+\.ffn_.*_exps=CPU`. `-cmoe/--cpu-moe` puts all experts on the CPU. `-ncmoe/--n-cpu-moe N` puts the experts of the first N layers on the CPU. These are MoE-only, so they do nothing for the dense 27B (already noted in `docs/architecture.md`).
+- `--moe-cache-mib N`: an LRU cache in VRAM for host-resident experts, merged in [#29887](https://github.com/ggml-org/llama.cpp/pull/29887) (2026-10-07). On Flash-Next Q4_0 the PR measured 4090 25.0 → 39.4 t/s and 5090 30.8 → 67.8 t/s, on an EPYC host with RAM for all experts. "Small batches only (<= 32 tokens)".
+- `-lzm/--lazy-mode {on,auto,off}`: "read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)". `auto` applies to tensors over 4 GiB, which **covers the 28.8 GB PLE table by default** (renamed in [#27969](https://github.com/ggml-org/llama.cpp/pull/27969)). Per [#30772](https://github.com/ggml-org/llama.cpp/pull/30772) (closed unmerged), mapped PLE pages still accumulate in RSS and single-threaded `madvise` can bottleneck NVMe.
+- `-lm/--load-mode {auto,none,mmap,…}`. With mmap, a tensor that fits nowhere is paged from disk implicitly.
+- **Pipeline parallelism turns off as soon as any tensor override is used** (`!has_tensor_overrides()` in `llama-context.cpp`, as described in the open PR [#29963](https://github.com/ggml-org/llama.cpp/pull/29963)). So `-ot`/`-ncmoe` plus two GPUs also loses the layer-split prefill overlap.
+
+**Numbers for this box:**
+
+- **Dense 27B:** no useful offload. Every weight is read every token, and DDR4 is ~51 GB/s against ~440 GB/s VRAM. Each GB moved to RAM costs ~20-25 ms/token on the CPU (estimate). With two cards the Q8_K_XL fits in VRAM anyway. The only RAM tier that helps is `--cache-ram` (prompt cache), as recorded in [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md).
+- **Flash-Next UD-Q4_K_XL, if RAM were free (hypothetical):** ~22 GB of experts on CPU = 29 % of experts, ~0.43 GB read per token from DDR4 at ~40 GB/s ≈ 11 ms. The GPU part ≈ (4.8 + 1.07) / 440 ≈ 13 ms. Layers are sequential, so the parts add up: **~24 ms ≈ ~40 t/s**, more with `--moe-cache-mib` (estimate). Prefill uploads the host experts once per ubatch: 22 GB over PCIe 4.0 x16 at ~25 GB/s ≈ 0.9 s per ubatch, so ~18 s of transfer for a 40k-token prompt at `-ub 2048` (estimate).
+- **Flash-Next UD-Q4_K_XL, on this box:** RAM is full (§1). Even with `--cache-ram` cut to near zero, 32 GB minus OS, OmniRoute (up to 10 GB) and qdrant leaves maybe ~15 GB (estimate), less than the ~22 GB needed. The rest is paged from the CPU-attached Gen4 x4 NVMe. At ~0.43 GB/token and an assumed 3-6 GB/s for ~1-3 MB expert reads, that is **~70-140 ms/token, ~6-12 t/s**. Prefill re-reads ~22 GB per ubatch from disk, so **minutes per 40k-token prompt**. The page cache will also compete with the swap that is already happening. An outside data point: an RX 9060 XT 16 GB with 64 GB DDR4-3200 got "8 t/s on short context. Longer context 3 t/s" on llama.cpp ROCm ([#29149 comment](https://github.com/ggml-org/llama.cpp/issues/29149)). **Not viable.**
+- **PLE table on SSD (any Flash-Next quant):** 32 rows per token (#30772) of 90 B each (Strata `AMD_HIP.md`), each on its own 4 KB page, so ≤128 KB of random IO per token. At 50 t/s that is a few thousand IOPS, trivial. Prefill touches ~32 pages per token, ~1.3 M random reads for 40k tokens. On NVMe at high queue depth that is a few seconds, but single-threaded `madvise` (#30772) can make it much slower. Strata measured +22.5 % prefill with the table in RAM against direct reads (§6). Decode impact: low. Prefill: measure it.
+
+## 6. Strata
+
+**Identity.** [github.com/Niko1221/Strata](https://github.com/Niko1221/Strata): "Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost". C++, **MIT** ([LICENSE](https://github.com/Niko1221/Strata/blob/main/LICENSE)). Its `docs/HOW_IT_WORKS.md` says it is "Built with parts of llama.cpp / ggml (MIT)". It supports Qwen3.8-Flash-Next (ISTA-DASLab GSQ-RCO quants recommended, Unsloth UD-IQ4_XS…Q8_0 supported), Qwen3.6-35B-A3B and DeepSeek V4.1-Flash. **There is no dense Qwen3.8-27B support**, so it cannot replace today's model.
+
+**How it tiers** ([`docs/HOW_IT_WORKS.md`](https://github.com/Niko1221/Strata/blob/main/docs/HOW_IT_WORKS.md), [`docs/MODELS.md`](https://github.com/Niko1221/Strata/blob/main/docs/MODELS.md)):
+
+- **VRAM:** attention/DeltaNet, routers, shared experts, output head, MTP, KV ("from 64K: only its most-read part, the rest streams from RAM"), plus an **expert cache** that fills the rest of VRAM with the most-used experts and adapts at runtime.
+- **RAM:** "all 24,576 experts, pinned". The CPU computes cache misses in place, concurrently with the GPU (AVX-512/AVX2). `--pcie-frac` splits misses between PCIe fetch and CPU compute. On gfx12 it defaults to 0.15 after a measured +9.9 % to +25.6 % decode on an R9700 (`docs/AMD_HIP.md`).
+- **SSD:** the 28.8 GB n-gram table, "read a few rows per token through the OS cache". From 0.2.7 it keeps the table in RAM when there is room for it plus 8 GiB.
+- **Low-RAM mode:** "setup maps them from the model files and keeps in RAM only what the card does not hold … With a small card most experts come from the SSD and it is much slower".
+- **Multi-GPU** ([`docs/MULTI_GPU.md`](https://github.com/Niko1221/Strata/blob/main/docs/MULTI_GPU.md)): pipeline (layer) split, "not tensor parallelism". "No NVLink or peer-to-peer access is needed, cards on x4 or x1 slots work". Each card caches the experts of its own layers, so two cards hold about twice as many experts. `docs/AMD_HIP.md` documents HIP layer split for gfx1201 pairs and adds "A split is worth it when no single card holds the model's experts".
+
+**AMD/ROCm:** the R9700 (gfx1201) is "Maintainer-tested", with Linux `hip-rx9070` engine packages in every recent release ([releases](https://github.com/Niko1221/Strata/releases)).
+
+**RAM requirement for the requested file:** `docs/MODELS.md` lists UD-Q4_K_XL as "111 GB download, 72 GiB experts, **RAM for all experts ~88 GB**". With less RAM it "keeps as many as fit and reads the rest from the SSD (much slower)".
+
+**Claimed vs verifiable:**
+
+| Claim | Source | Host RAM | Verifiable? |
+|---|---|---|---|
+| UD-Q4_K_XL on one R9700: 63 t/s prose, 87 code, 32K prompt 1,110-1,320 t/s | `docs/MODELS.md`, "engines before 0.1.45.3" | not stated there | Self-reported |
+| IQ3_S on one R9700: 87.6-101.9 t/s across 1K-128K | [community R9700 report](https://github.com/Niko1221/Strata/blob/main/bench/results/2026-10-08-community-r9700-linux/README.md) | **~503 GiB**, 2x EPYC 9334 | Has per-request JSON and hashes. "Single GPU still uses CPU experts, system RAM and PCIe; these rates do not establish desktop-CPU speed or minimum RAM" |
+| R9700 + 9070 XT, 47 GB RAM, Coder IQ1_M: 45.5 / 48.3 t/s decode | `docs/AMD_HIP.md` | 47 GB | Self-reported |
+| "125B+ MoE on a single 12GB GPU", "25-38 tok/s" | [stratallm.org](https://stratallm.org/) | n/a | The site calls itself "Unofficial Community Documentation". Its speed figures contradict each other and link no method. Ignore it |
+
+None of these numbers were rerun for this doc. The project says its benchmark protocol "is still a draft" (`docs/COMMUNITY_BENCHMARKS.md`).
+
+**Maturity:** repo created 2026-09-24. ~4,690 commits, ~3,594 of them by the one maintainer. ~36k stars, 895 open issues, a release roughly daily (0.1.40 → 0.2.7 in three weeks). Installation is `curl … | sh`, an AppImage, or Docker (`docs/DOCKER.md`, not read). It is young, single-maintainer and churning, and it would need its own container and gateway wiring here.
+
+**Verdict for this box:** not worth it now. The 32 GB of RAM defeats its core design (all experts pinned in RAM). Low-RAM mode on 2x32 GB VRAM would still read ~20 GB of experts from NVMe for UD-Q4_K_XL. It cannot run the dense 27B that this stack is tuned around. **Revisit if RAM goes to 96-128 GB** (the board takes 128 GB). On a big-RAM host its self-reported R9700 Flash-Next numbers (60-100 t/s) are well above llama.cpp's 27B, so it would be worth a real A/B then.
+
+## 7. Recommendation
+
+**Primary: keep `Qwen3.8-27B-UD-Q4_K_XL` + DFlash2 and run two single-GPU replicas, one per card.**
+
+Why:
+
+- (1) Layer split gives a dense model no decode gain.
+- (2) Tensor split is experimental on ROCm, measured at 1.31x, crashes with DFlash2, and would run over a PCIe 3.0 x4 chipset link.
+- (3) Q8_K_XL roughly halves decode.
+- (4) Flash-Next at a usable quant does not fit, and RAM offload pages from NVMe.
+- (5) The workload is 2 parallel agents. The log already shows decode falling to p10 15 t/s while the other slot prefills ([`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md)). Two replicas remove that interference and double each slot's context.
+
+Per-instance flags (two services, identical except the device). Each is today's command with these changes:
+
+```text
+--device ROCm0               # ROCm1 for the second service; confirm names with --list-devices
+--n-gpu-layers 999
+--ctx-size 262144 --parallel 1   # full 262144 per agent instead of 131072
+--flash-attn on --cache-type-k q8_0 --cache-type-v q8_0
+--cache-ram 6144             # per instance; both together must stay well under free RAM (§1)
+--spec-type draft-dflash -md /models/Qwen3.8-27B-DFlash2-Q4_K_M.gguf --spec-draft-n-max 7
+--jinja
+```
+
+`GPU_MAX_HW_QUEUES=1` stays on both services (`docs/architecture.md#gpu-access`). Gateway: register both backends in OmniRoute and make sure a conversation stays on one backend (prompt-cache affinity). Otherwise every switch is a full re-prefill. Whether OmniRoute can balance two connections per conversation was not checked.
+
+**Independently of the second card: lower `LLAMA_CACHE_RAM`.** 24 GiB on a 32 GB host with a 10 GB OmniRoute limit oversubscribes RAM.
+
+**Alternatives to A/B, in order:**
+
+1. **Single fast stream:** Q4_K_XL with `-sm tensor -ts 1,1 -fa on --fit off --ctx-size <set by hand> --spec-type draft-mtp --spec-draft-n-max 2` (the built-in MTP head; DFlash2 crashes with `tensor`). Estimate ~34 t/s no spec, ~50 with MTP if stable. Watch for #27122-style lockups.
+2. **Quality:** Q8_K_XL (or UD-Q6_K_XL at 25.3 GB) with `-sm layer`, DFlash2 placed with `-devd`, ~25 t/s estimated. Only if Q4 quality is shown to fail on your tasks.
+3. **Flash-Next experiment:** UD-IQ3_XXS or UD-Q2_K_XL, all in VRAM: `-sm layer -ngl 999 --lazy-mode auto --load-mode none -fa on -c 131072`, MTP via `-md MTP/mtp-Qwen3.8-Flash-Next-shared-Q8_0.gguf --spec-type draft-mtp` once #29811/#30860 are fixed. Judge it on your own agent tasks. The KLD at these sizes is 3.5-5x that of UD-Q4_K_XL.
+
+## 8. What to benchmark once the card arrives
+
+1. **Topology:** `sudo lspci -vv` LnkSta for both cards (expect 16 GT/s x16 and 8 GT/s x4). Run `rocm-smi --showtopo` and `rocm-bandwidth-test` for host↔GPU and GPU↔GPU (does P2P work through the chipset?). Check the PSU and the case slot fit first.
+2. **Per-card baseline:** `llama-bench -m Qwen3.8-27B-UD-Q4_K_XL.gguf -dev ROCm0` and then `-dev ROCm1`, `-fa 1 -ctk q8_0 -ctv q8_0 -p 512,8192 -n 128 -d 0,32768`. Decode should match. Load time and prompt-cache restores are where the x4 link shows.
+3. **Split modes:** the same bench with `-sm layer` vs `-sm tensor -ts 1,1` for Q4_K_XL and Q8_K_XL. Compare tg against #28863's 1.31x and pp against layer.
+4. **Server replay** (fixed ~40k and ~2k prompts, production temperature, concurrency 1 and 2) for: (a) two replicas, (b) one `-sm layer --parallel 2`, (c) `-sm tensor` + MTP. Record `timings.predicted_per_second`, `prompt_per_second`, draft acceptance, and the p10 decode under concurrency.
+5. **Correctness:** a needle test at ~6-10k and ~60-90k tokens on each config, because #30884 reproduces on 2x R9700 with `--tensor-split`. Also check that DFlash2 output under `-sm layer` matches single-card output on a fixed prompt.
+6. **RAM:** `free -h` and `vmstat 5` after an hour of agent traffic, before and after lowering `--cache-ram`. Swap-in should be ~0.
+7. **Optional Flash-Next:** that UD-IQ3_XXS/UD-Q2_K_XL loads at all on HIP (#29149, try `--load-mode none`), tg/pp at 4k/32k, and prefill time with the PLE table on NVMe.
+
+## Open questions
+
+- Does OmniRoute support two backends with sticky routing per conversation?
+- Does the `ghcr.io/ggml-org/llama.cpp:server-rocm` image build with `GGML_HIP_RCCL`? It only matters for `-sm tensor`.
+- Does ROCm P2P work between a CPU-attached slot and a chipset-attached slot on B550? It is unverified, and the answer probably does not matter outside `tensor` mode.
+- Is there a B550/X570 board with CPU x8/x8 bifurcation that fits this chassis? It would give both cards Gen4 x8. It is only worth it if step 3 shows `tensor` scaling well. No specific board was checked.
+
+## Sources
+
+- Hugging Face: [unsloth/Qwen3.8-27B-GGUF tree](https://huggingface.co/api/models/unsloth/Qwen3.8-27B-GGUF/tree/main), [its config.json](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/raw/main/config.json), [UD-Q8_K_XL file](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF?show_file_info=Qwen3.8-27B-UD-Q8_K_XL.gguf), [unsloth/Qwen3.8-Flash-Next-GGUF](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF) and its [tree](https://huggingface.co/api/models/unsloth/Qwen3.8-Flash-Next-GGUF/tree/main?recursive=true), [UD-Q4_K_XL shard 1](https://huggingface.co/unsloth/Qwen3.8-Flash-Next-GGUF?show_file_info=UD-Q4_K_XL%2FQwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf), [z-lab DFlash2 GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF), [incoai Flash-Next DFlash2 GGUF](https://huggingface.co/incoai/Qwen3.8-Flash-Next-DFlash2-GGUF). Tensor sizes come from parsing the GGUF headers (range requests) with a throwaway script.
+- Unsloth: [Qwen3.8-Flash-Next run guide](https://unsloth.ai/docs/models/qwen3.8-next) (sizes, KLD table, MTP flags).
+- llama.cpp master (fetched 2026-10-29): [`docs/multi-gpu.md`](https://github.com/ggml-org/llama.cpp/blob/master/docs/multi-gpu.md), [`common/arg.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/common/arg.cpp), [`src/llama-arch.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/src/llama-arch.cpp), [`src/llama-context.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/src/llama-context.cpp). PRs: [#19378](https://github.com/ggml-org/llama.cpp/pull/19378), [#23792](https://github.com/ggml-org/llama.cpp/pull/23792), [#27342](https://github.com/ggml-org/llama.cpp/pull/27342), [#27742](https://github.com/ggml-org/llama.cpp/pull/27742), [#27825](https://github.com/ggml-org/llama.cpp/pull/27825), [#27858](https://github.com/ggml-org/llama.cpp/pull/27858), [#27861](https://github.com/ggml-org/llama.cpp/pull/27861), [#27969](https://github.com/ggml-org/llama.cpp/pull/27969), [#28569](https://github.com/ggml-org/llama.cpp/pull/28569), [#29761](https://github.com/ggml-org/llama.cpp/pull/29761), [#29887](https://github.com/ggml-org/llama.cpp/pull/29887), [#29963](https://github.com/ggml-org/llama.cpp/pull/29963), [#30772](https://github.com/ggml-org/llama.cpp/pull/30772), [#30815](https://github.com/ggml-org/llama.cpp/pull/30815), [#31243](https://github.com/ggml-org/llama.cpp/pull/31243). Issues: [#23567](https://github.com/ggml-org/llama.cpp/issues/23567), [#27122](https://github.com/ggml-org/llama.cpp/issues/27122), [#27829](https://github.com/ggml-org/llama.cpp/issues/27829), [#28582](https://github.com/ggml-org/llama.cpp/issues/28582), [#28777](https://github.com/ggml-org/llama.cpp/issues/28777), [#28863](https://github.com/ggml-org/llama.cpp/issues/28863), [#29149](https://github.com/ggml-org/llama.cpp/issues/29149), [#29811](https://github.com/ggml-org/llama.cpp/issues/29811), [#30860](https://github.com/ggml-org/llama.cpp/issues/30860), [#30884](https://github.com/ggml-org/llama.cpp/issues/30884), [#30887](https://github.com/ggml-org/llama.cpp/issues/30887).
+- Hardware: [ASRock B550M Pro4 manual](https://download.asrock.com/Manual/B550M%20Pro4.pdf) (asrock.com's spec page returned no content to the fetcher), [ComputerBase B550](https://www.computerbase.de/2020-06/amd-b550-chipsatz-mainboard-test/), [Guru3D B550 PCIe](https://www.guru3d.com/review/amd-ryzen-3-3100-and-3300x-review/page-3/), [Puget R9700](https://www.pugetsystems.com/parts/Video-Card/AMD-Radeon-AI-PRO-R9700-32GB-16908/), [Tom's Hardware R9700](https://www.tomshardware.com/pc-components/gpus/amd-launches-radeon-ai-pro-r9700-to-challenge-nvidias-ai-market-dominance).
+- Strata: [repo](https://github.com/Niko1221/Strata) (README, LICENSE, `docs/HOW_IT_WORKS.md`, `MODELS.md`, `MULTI_GPU.md`, `SECOND_GPU.md`, `AMD_HIP.md`, `COMMUNITY_BENCHMARKS.md`, read via the GitHub API on 2026-10-29), [releases](https://github.com/Niko1221/Strata/releases), [stratallm.org](https://stratallm.org/).
+- This repo: `docker-compose.yml`, `.env.example`, `docs/architecture.md`, [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md), [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md), [`500k-context-2-parallel-agents-gpu-roadmap.md`](500k-context-2-parallel-agents-gpu-roadmap.md).

docs/research/inference-engine-spec-decoding-multi-gpu.md

@@ -0,0 +1,144 @@
+# Research: which inference engine for better drafting now and two-GPU splitting later (plus the `--cache-ram` typo bug)
+
+**Date:** 2026-11-02
+
+**Question:** llama.cpp is said to lag on newer drafters (DFlash/DSpark, EAGLE-3/EAGLE-4) and its multi-GPU tensor parallelism is experimental. Which engine fits the server best for speed and efficiency, with a better drafter now and the main model split across two R9700s later? Constraints: commercial use must be allowed (engine and weights), two agents sometimes share the backend, there is an overnight backlog of API calls (throughput matters, not only single-stream latency), and context stays as it is. Second question: why does the main model misspell words when the RAM prompt cache (`--cache-ram`) is on?
+
+**Scope note:** research only. Nothing in compose, `.env.example` or `models.yml` was changed. Verified facts carry a source link. Lines marked **(inference)** are my reasoning, not something a source states. Engines and drafters move weekly in this area, so anything here older than about a month should be re-checked.
+
+## Answer, short version
+
+**Now (one R9700): stay on llama.cpp.** The premise is partly out of date. llama.cpp master has `draft-eagle3`, `draft-dflash`, `draft-dspark` and `draft-mtp` as speculative types ([docs/speculative.md](https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md), [common/speculative.cpp](https://github.com/ggml-org/llama.cpp/blob/master/common/speculative.cpp)), and DFlash2 was merged on 2026-08-27 ([#27342](https://github.com/ggml-org/llama.cpp/pull/27342)). No engine on a single R9700 has a measured result that beats the current setup for this model:
+
+- SGLang does not build for gfx1201 on `main` ([sglang#46900](https://github.com/sgl-project/sglang/issues/46900)).
+- Upstream vLLM lists gfx1201, but nobody has published single-R9700 numbers for Qwen3.8-27B. Its official FP8 checkpoint (~28.5 GB, per the [SGLang cookbook](https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.8-27B)) doesn't fit beside a usable KV cache on 32 GB.
+- EAGLE-4 isn't in llama.cpp and its license limits commercial use (below).
+
+Cheap wins on llama.cpp:
+
+1. **Keep `LLAMA_CACHE_RAM=0`.** That is the workaround for the typo bug until [#27624](https://github.com/ggml-org/llama.cpp/pull/27624) merges (diagnosis below).
+2. A/B **`draft-dspark` with `RedHatAI/Qwen3.8-27B-speculator.dspark`** (Apache-2.0) against DFlash2. It is an option only: no GGUF of it exists yet, and conversion is unverified for this target.
+3. Watch two open, R9700-specific llama.cpp performance issues. Both hit exactly this workload, speculative verify batches with two slots: [#28867](https://github.com/ggml-org/llama.cpp/issues/28867) (~20 % spec-decode loss from a flash-attention dispatch threshold) and [#29820](https://github.com/ggml-org/llama.cpp/issues/29820) (up to +44 % from an MMVQ batch-size change).
+
+**When GPU #2 arrives: move the main model to vLLM with tensor parallel 2.** The only real-world TP2 evidence on 2× R9700 for this exact model is vLLM. One publicly documented setup serves `Qwen/Qwen3.8-27B-FP8` with MTP-3 at 262k context and reaches 66 tok/s single-stream and ~1,014 tok/s aggregate at 32 concurrent sequences ([ROCm/vllm#1229](https://github.com/ROCm/vllm/issues/1229)). That is the batch throughput the overnight backlog needs. llama.cpp's `--split-mode tensor` is labelled EXPERIMENTAL in its own help text ([common/arg.cpp](https://github.com/ggml-org/llama.cpp/blob/master/common/arg.cpp)), and it crashes with DFlash drafters today ([#27833](https://github.com/ggml-org/llama.cpp/issues/27833), [#28777](https://github.com/ggml-org/llama.cpp/issues/28777)). Use MTP-3 (built into the checkpoint) as the default, because it scales with concurrency. DFlash2 wins only at 1-4 users ([RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)). Every engine and weight file in that path is MIT or Apache-2.0.
+
+**Avoid for work use:** EAGLE-4 (source-available, commercial use only below USD 20M aggregate revenue of the licensee and its affiliates, [LICENSE](https://github.com/eagle-hq/EAGLE-4/blob/main/LICENSE)) and the `RadixArk/Qwen3.8-27B-DSpark` drafter (license `other`, no terms in the card, [model card](https://huggingface.co/RadixArk/Qwen3.8-27B-DSpark)).
+
+## 1. Current state (from this repo)
+
+`docker-compose.yml` → `llama-server`: `ghcr.io/ggml-org/llama.cpp:server-rocm`, `--parallel 2`, `--kv-unified`, `--flash-attn on`, `--cache-type-k/v ${LLAMA_KV_CACHE_TYPE}`, `--cache-ram ${LLAMA_CACHE_RAM}`, `${LLAMA_SPEC_ARGS}`, `--jinja`. `.env.example` sets `LLAMA_CACHE_RAM=0` and `LLAMA_SPEC_ARGS` to DFlash2 (Q8_0 drafter, `--spec-draft-n-max 7`). Earlier measurements in [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md) found 45.9 tok/s weighted decode, 50 % draft acceptance, and prefill of ~950 tok/s at 40-50k tokens.
+
+Note: compose's fallback for `--cache-ram` is still `24576`, while `.env.example` says `0`. The server's `.env` wins, but if the key were ever missing, the cache would silently come back on. Not changed here.
+
+Model facts that drive everything below, from [Qwen/Qwen3.8-27B config.json](https://huggingface.co/Qwen/Qwen3.8-27B/raw/main/config.json):
+
+- 64 layers with `full_attention_interval: 4`, so 16 attention layers and 48 Gated DeltaNet (recurrent) layers.
+- `mtp_num_hidden_layers: 1`, so the official weights **ship an MTP head**.
+- License Apache-2.0 ([model card](https://huggingface.co/Qwen/Qwen3.8-27B)).
+
+## 2. Comparison table
+
+| Engine | RDNA4 / gfx1201 (R9700) | Spec decoding for Qwen3.8-27B | Multi-GPU | Prefix cache (hybrid model) | Batch throughput | Engine license |
+|---|---|---|---|---|---|---|
+| **llama.cpp** (current) | Official ROCm and Vulkan builds. Open R9700 correctness/perf issues: [#30884](https://github.com/ggml-org/llama.cpp/issues/30884) (FA long-context EOS on Windows), [#28867](https://github.com/ggml-org/llama.cpp/issues/28867), [#29820](https://github.com/ggml-org/llama.cpp/issues/29820) | `draft-mtp`, `draft-dflash` (DFlash2), `draft-dspark`, `draft-eagle3`, n-gram types ([docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md)). No EAGLE-4. No Qwen3.8-27B EAGLE-3 drafter found on HF | `-sm layer` (default, pipelined: more VRAM, no single-stream speed-up), `row`, `tensor` (EXPERIMENTAL). DFlash + `-sm tensor` crashes ([#27833](https://github.com/ggml-org/llama.cpp/issues/27833), [#28777](https://github.com/ggml-org/llama.cpp/issues/28777)). Repeated-garbage bug on CUDA ([#30075](https://github.com/ggml-org/llama.cpp/issues/30075)) | Per-slot prompt plus context checkpoints, plus `--cache-ram` RAM tier, with open correctness bugs (§4) | Fixed slots. Context is divided per slot. DFlash acceptance collapses at high slot counts ([#27117](https://github.com/ggml-org/llama.cpp/issues/27117)) | MIT ([LICENSE](https://github.com/ggml-org/llama.cpp/blob/master/LICENSE)) |
+| **vLLM** (upstream) | Listed: "Radeon RX 9000 series (gfx1200/1201)", ROCm image built for gfx1201 ([install docs](https://docs.vllm.ai/en/latest/getting_started/installation/gpu/), [Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base)). AITER and LMCache are built for gfx942/950 only (same Dockerfile). No CI pool covers gfx11/gfx12 ([#64614](https://github.com/vllm-project/vllm/pull/64614)) | `mtp`/`qwen3_5_mtp`, `dflash`, `dspark`, `eagle`, `eagle3`, `draft_model`, `ngram`, `suffix` ([speculative.py](https://github.com/vllm-project/vllm/blob/main/vllm/config/speculative.py)). Recipe: DFlash2 needs ≥0.28.0 ([recipe](https://recipes.vllm.ai/Qwen/Qwen3.8-27B)). Stock vLLM fails to load an FP8 DFlash drafter ([RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)) | Tensor parallel, the mature path. TP2 on 2× R9700 in production ([ROCm/vllm#1229](https://github.com/ROCm/vllm/issues/1229)). `NCCL_PROTO=Simple` recommended for PCIe | Hybrid-aware: `mamba_cache_mode="align"` caches recurrent state at block boundaries ([cache.py](https://github.com/vllm-project/vllm/blob/main/vllm/config/cache.py)). Optional CPU KV offload (`kv_offloading_size`, same file) | Paged KV shared by all sequences, continuous batching. 1,014 tok/s aggregate at c32 on 2× R9700 ([#1229](https://github.com/ROCm/vllm/issues/1229)) | Apache-2.0 ([LICENSE](https://github.com/vllm-project/vllm/blob/main/LICENSE)) |
+| **vLLM RDNA4 fork** ([Capicua25x/vllm-rocm-rdna4](https://github.com/Capicua25x/vllm-rocm-rdna4)) | Built for gfx1200/1201. Validated on gfx1201 TP2 only. Based on 0.28.0 (rc13, 2026-09-03), while upstream is at v0.33.0rc3 (2026-11-02) | MTP-3 (default), DFlash2 including FP8 drafters (fix `50f7a7d12`) | TP2 measured | `align` mode, fp8 KV plus bf16 SSM state | MTP-3: 315 agg @c16 on 6k prompts. DFlash2 is ahead only to ~c4 | Apache-2.0 (one-person fork) |
+| **SGLang** | **Not supported on `main`**. RDNA fixes go to the `amd_march` branch until RDNA CI exists. As of 2026-10-30 `main` fails to build sgl-kernel for gfx1201 ([#46900](https://github.com/sgl-project/sglang/issues/46900), [#30599](https://github.com/sgl-project/sglang/issues/30599)) | EAGLE/NEXTN (MTP), DFLASH, DSPARK ([cookbook](https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.8-27B)). EAGLE-4 via `eagle-harness` (CUDA B300, MI355X only) | TP mature on CUDA/CDNA | Radix cache with mamba strategies (cookbook) | Strong on supported hardware | Apache-2.0 ([LICENSE](https://github.com/sgl-project/sglang/blob/main/LICENSE)) |
+| TGI | n/a | n/a | n/a | n/a | n/a | Repo archived, "maintenance mode", recommends vLLM/SGLang/llama.cpp ([README](https://github.com/huggingface/text-generation-inference)) |
+| ExLlamaV3 / TabbyAPI | CUDA only (install requires CUDA ≥12.4, [README](https://github.com/turboderp-org/exllamav3)) | n/a | n/a | n/a | n/a | MIT |
+
+Lemonade ([lemonade-sdk/lemonade](https://github.com/lemonade-sdk/lemonade), Apache-2.0) and LM Studio/Ollama wrap llama.cpp for GGUF. They add no drafter or tensor-parallel capability beyond llama.cpp, so they are not evaluated further **(inference from their being llama.cpp front-ends)**.
+
+## 3. Drafters and weights: what exists, and can it be used at work?
+
+| Artifact | What | Engine support | License |
+|---|---|---|---|
+| `Qwen/Qwen3.8-27B` | Target, BF16, MTP head included | all | Apache-2.0 ([HF](https://huggingface.co/Qwen/Qwen3.8-27B)) |
+| `Qwen/Qwen3.8-27B-FP8` | Official FP8, ~28.5 GB | vLLM, SGLang | Apache-2.0 ([HF](https://huggingface.co/Qwen/Qwen3.8-27B-FP8)) |
+| `unsloth/Qwen3.8-27B-GGUF` | Current GGUF, plus `MTP/mtp-Qwen3.8-27B-Q4_0.gguf` | llama.cpp | Apache-2.0 ([HF](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF)) |
+| `amd/Qwen3.8-27B-Quark-AWQ-INT4-W4A16`, `RedHatAI/Qwen3.8-27B-INT4` | INT4 W4A16, ~19.5 GB each | vLLM (RDNA hybrid W4A16 kernel, below) | Apache-2.0 ([AMD](https://huggingface.co/amd/Qwen3.8-27B-Quark-AWQ-INT4-W4A16), [RedHat](https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4)) |
+| `z-lab/Qwen3.8-27B-DFlash2` (+ `-GGUF`) | DFlash2 drafter, 3.85 GB bf16, the current one | llama.cpp, vLLM, SGLang | Apache-2.0 ([HF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2), [GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF)) |
+| `incoai/Qwen3.8-27B-DFlash2` | DFlash2, used in the vLLM and SGLang recipes | vLLM, SGLang | Apache-2.0 ([HF](https://huggingface.co/incoai/Qwen3.8-27B-DFlash2)) |
+| `RedHatAI/Qwen3.8-27B-speculator.dspark` | DSpark, block 8, confidence head, 3.98 GB | vLLM (card). llama.cpp converts speculators-format DSpark ([docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md)), but this target is **untested** | Apache-2.0 ([HF](https://huggingface.co/RedHatAI/Qwen3.8-27B-speculator.dspark)) |
+| `RadixArk/Qwen3.8-27B-DSpark` | "DSpark v2", 1.86B | SGLang | `other`, no terms in card: **do not use until clarified** ([HF](https://huggingface.co/RadixArk/Qwen3.8-27B-DSpark)) |
+| `EAGLE-Corp/Qwen3.8-27B-EAGLE4` | EAGLE-4. Claims 3.59× at T=0 vs 2.80× DFlash2 (4× B300, c1) | `eagle-harness` for SGLang/vLLM/TokenSpeed on B300 and MI355X ([README](https://github.com/eagle-hq/EAGLE-4)). Not llama.cpp, not RDNA | Source-available, commercial use only while licensee plus affiliates are under USD 20M revenue per 12 months, and no "Competing Offering" ([LICENSE §3](https://github.com/eagle-hq/EAGLE-4/blob/main/LICENSE)). **Not suitable for work use without legal review** |
+
+Speculation gains shrink with concurrency for every method. EAGLE-4's own table drops from 3.42× at 1 request to 1.39× at 64 on SGLang ([README](https://github.com/eagle-hq/EAGLE-4)). On 2× R9700, DFlash2 gives +18-26 % single-stream over MTP-3 but falls behind it from c8 up ([RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)). For the overnight backlog, the drafter matters less than paged batching.
+
+## 4. Diagnosis: misspellings with `--cache-ram` on
+
+### What's verified upstream
+
+1. **[#27148](https://github.com/ggml-org/llama.cpp/issues/27148) (open): the RAM prompt cache restores state from an unrelated conversation into a slot.** When `get_available_slot()` falls back to an LRU victim, `server_prompt_cache::load()` returns `true` even when nothing matched. `prompt_clear()` then never runs, so the victim slot keeps its previous owner's KV and recurrent state. The issue has independent reproductions:
+   - on Qwen3.8-27B (hybrid GDN plus attention) with `--kv-unified` on Metal, where responses "switched mid-sentence into another request's output"
+   - with strictly sequential single-user requests on gfx1151 ROCm with `kv_unified = true`
+
+   Mitigation from the thread: with `kv_unified = true`, `--cache-ram 0` **or** `--no-cache-idle-slots` closes it. With `kv_unified = false`, both are needed.
+2. **Fix PR [#27624](https://github.com/ggml-org/llama.cpp/pull/27624) (open):** it replaces the boolean with `RESTORED`/`NO_MATCH`/`FAILED`, and clears an LRU victim when restore fails, nothing matches, or `cache_ram=0`. The PR says the `cache_ram=0` path has the same missing clear.
+3. **Related hardening [#28058](https://github.com/ggml-org/llama.cpp/pull/28058) (open):** `llama_memory_recurrent::seq_rm` returned `true` for a partial erase it cannot do, "leaving the removed tokens in the state", so "stale recurrent state leaked into the new prompt positions on hybrid/recurrent models". The PR's main fix targets integrated GPUs and doesn't apply to the R9700. The `seq_rm` part applies to any hybrid model.
+4. **[#30961](https://github.com/ggml-org/llama.cpp/issues/30961) / [#30997](https://github.com/ggml-org/llama.cpp/pull/30997) (both open):** with `--kv-unified` plus `--cache-ram` plus ≥2 slots, a cached prompt fails to restore while another idle slot still holds its cells. That causes a full re-prefill (performance, not corruption). Same config as this stack.
+5. Merged related work, so the build date matters: [#30498](https://github.com/ggml-org/llama.cpp/pull/30498), [#30567](https://github.com/ggml-org/llama.cpp/pull/30567) and [#30574](https://github.com/ggml-org/llama.cpp/pull/30574) (2026-10-15, cache bookkeeping and saving the draft state). [#26004](https://github.com/ggml-org/llama.cpp/pull/26004) (2026-10-08, checkpoints across slot save/restore). None of them fixes #27148.
+
+### Most likely cause here (inference)
+
+The stack runs exactly the affected configuration: hybrid model, `--parallel 2`, `--kv-unified`, prompt cache on. On a pure-attention model, a stale slot shows up as "answers the wrong conversation". On this model, 48 of 64 layers carry a recurrent state that cannot be partially rolled back. Leftover recurrent state from the previous owner, mixed with correct attention KV for the new prompt, would plausibly give output that stays on topic but is subtly corrupted at the token level. That matches "misspells a lot" better than a wholesale swap.
+
+Two things argue against the alternatives:
+
+- **The drafter is not the cause.** The target verifies every draft token, so a stale or mismatched draft state only lowers acceptance ([#30574](https://github.com/ggml-org/llama.cpp/pull/30574) shows exactly that: acceptance dropped, "the output is identical"). Speculative verification on GDN targets is not bit-identical to plain decoding ([#31472](https://github.com/ggml-org/llama.cpp/issues/31472)), but that shows up as paraphrase-level drift with or without the RAM cache.
+- **Quantized KV alone is unlikely.** The symptom tracks `--cache-ram` on/off at a fixed `q8_0`.
+
+Rule out before blaming llama.cpp: in a similar "dropped characters with prompt cache" report on a hybrid Qwen, the root cause was the client's SSE parser stripping repeated deltas ([#21681](https://github.com/ggml-org/llama.cpp/issues/21681)). Check that the typos are present in the raw stream from `llama-server`, not just in what the gateway or Coding CLI shows.
+
+### Workaround and how to confirm
+
+- **Workaround (already in `.env.example`): `LLAMA_CACHE_RAM=0`.** Per the #27148 thread it is sufficient with `--kv-unified`. The cost is the re-prefill that the RAM cache used to save, ~47 s per 45k-token prompt per [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md). Optionally add `--no-cache-idle-slots` as a second guard. It's harmless, since idle-slot caching needs the RAM cache anyway.
+- **Confirm the diagnosis (cheap):** reuse the #27148 / #27624 test shape. Send A (a short unique conversation), then B (an unrelated long prompt with a nonce), then C (A again with a new message), with `--cache-ram` on, `temperature 0`, and the drafter off. Compare C's output against a fresh-server cold run of the same prompt. A diff with cache on and none with cache off confirms it. Log lines to grep: `selected slot by LRU`, `prompt_load`, `failed to restore`.
+- **Re-enable the cache** only on a build that contains #27624 (and ideally #30997), and after the test above passes.
+
+## 5. Single GPU: why not switch to vLLM now?
+
+- **Quant formats.** vLLM's docs table marks AWQ and GPTQ as ❌ on "AMD GPU" ([quantization README](https://github.com/vllm-project/vllm/blob/main/docs/features/quantization/README.md)). That table is stale for RDNA: `HybridW4A16LinearKernel` for gfx11/gfx12 (AWQ, GPTQ, compressed-tensors INT4, group 32/64/128) merged on 2026-07-14 ([#40977](https://github.com/vllm-project/vllm/pull/40977)), with follow-up fixes still landing ([#64614](https://github.com/vllm-project/vllm/pull/64614) merged 2026-11-02, [#52637](https://github.com/vllm-project/vllm/pull/52637) open). GGUF in vLLM is "highly experimental and under-optimized" and moved to an out-of-tree plugin ([gguf.md](https://github.com/vllm-project/vllm/blob/main/docs/features/quantization/gguf.md)).
+- **No measurement exists.** I found no published single-R9700 vLLM numbers for Qwen3.8-27B INT4. The fork documents TP2 only.
+- **VRAM (inference, unmeasured).** INT4 weights are ~19.5 GB. Attention KV is 16 layers × 2 × 4 KV heads × 256 × 1 byte = 32 KiB/token at fp8. With MTP (no separate drafter) and ~0.92 memory utilisation, that leaves roughly 7 GB, about 230k tokens of KV shared across all sequences. A 3.85 GB DFlash2 drafter would cut that to roughly 120k. Today's 262k at `q8_0` would not clearly fit, and speed versus llama.cpp's 46 tok/s is unknown.
+- **ROCm-specific instability.** A vLLM 0.20 → 0.23 image upgrade halved FP8 throughput on 4× R9700, root cause unknown ([Level1Techs](https://forum.level1techs.com/t/vllm-0-23-x-rocm-7-14-upgrade-on-4x-radeon-ai-pro-r9700-50-regression-on-one-model-unaffected-on-another-but-a-free-9-via-kernel-tuning/253460)). AITER generated a kernel over the LDS limit on gfx1201 ([#48723](https://github.com/vllm-project/vllm/issues/48723), open). FP8 RowWise scaled MM is being disabled on RDNA4 ([#57861](https://github.com/vllm-project/vllm/pull/57861), open). A Quark `exclude` bug silently breaks MTP ([#65034](https://github.com/vllm-project/vllm/issues/65034), open).
+
+Verdict: a vLLM trial on one card is a reasonable side experiment (`RedHatAI/Qwen3.8-27B-INT4` plus `--speculative-config '{"method":"mtp","num_speculative_tokens":3}'`, as on [its card](https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4)), but not a replacement until it is benchmarked with `speed-bench` against the current numbers.
+
+## 6. Two GPUs: the plan
+
+1. Same backend on both cards. Both are R9700s on ROCm, as already noted in [`500k-context-2-parallel-agents-gpu-roadmap.md`](500k-context-2-parallel-agents-gpu-roadmap.md).
+2. vLLM TP2, `Qwen/Qwen3.8-27B-FP8` (or the INT4 checkpoint for more KV), `--enable-prefix-caching` (hybrid `align` mode), fp8 KV, MTP-3, `--tool-call-parser qwen3_coder --reasoning-parser qwen3` ([SGLang cookbook](https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.8-27B) names the same parsers, and [RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md) warns that a hermes parser "craters agentic scores"), and `NCCL_PROTO=Simple`.
+3. Start on upstream `vllm/vllm-openai-rocm`. Fall back to the fork's image only if upstream underperforms. The fork is a single maintainer, five minor versions behind, and its FP8-drafter fix isn't upstream.
+4. The overnight backlog benefits most: one shared KV pool, `--max-num-seqs` well above 2, and no per-slot context division. The two-agents-by-day case keeps the large window, because vLLM reserves KV per token used, not per slot.
+5. If llama.cpp's `-sm tensor` stabilises with DFlash (watch #27833, #28777), re-evaluate. Until then, llama.cpp on two cards means `-sm layer`, which buys VRAM (a bigger quant, more slots) but no single-stream speed.
+
+## 7. Migration risks (llama.cpp → vLLM)
+
+- **Weights and downloader.** vLLM loads a safetensors directory (18+ shards), not one GGUF. `models.yml`'s one-file-per-URL downloader would need many entries or a different fetch method. The HF cache volume grows by ~20-30 GB.
+- **Tool-call and reasoning parsing** change completely (vLLM parsers instead of llama.cpp's Jinja/PEG). Re-run the checks from [`qwen3.8-27b-tool-calling.md`](qwen3.8-27b-tool-calling.md) through the gateway. vLLM ignores OpenAI `reasoning_effort` for this family. Thinking is a `chat_template_kwargs` switch ([RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)).
+- **Startup and reproducibility.** First start compiles and autotunes kernels (minutes), and the result depends on timing: different compile caches give different temperature-0 outputs. Persist the compile-cache volume ([RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)).
+- **Gateway timeouts and slot assumptions.** OmniRoute prompt-cache affinity, the per-slot-context rule in [`architecture.md`](../architecture.md) and the Coding CLI context settings are all llama.cpp-specific. They need rewriting, not porting.
+- **ROCm regressions between image tags** (Level1Techs report above). Pin image digests and benchmark each bump.
+- **`GPU_MAX_HW_QUEUES` and numeric `group_add`** ([`architecture.md` → GPU access](../architecture.md#gpu-access)) still apply to any ROCm container. vLLM also needs `--ipc=host` or a large `shm_size` for TP workers ([fork run commands](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)).
+- **Rollback is cheap** if the llama.cpp service stays in compose under a profile.
+
+## 8. Open questions
+
+- Does the server's current `server-rocm` image already contain any of #30498/#30567 (2026-10-15)? `docker logs llama-server | head` shows the build.
+- Is DSpark conversion (`convert_hf_to_gguf.py --target-model-dir`) supported for a `Qwen3_5ForConditionalGeneration` verifier? The docs say "only drafts with a Qwen3 backbone". The RedHat draft's layers are Qwen3-style, but this is untested.
+- Single-R9700 vLLM INT4 numbers: decode, prefill at 40k, and the KV pool size printed at boot.
+- RadixArk DSpark license terms (ask the publisher).
+
+## Sources
+
+- Repo: [`docker-compose.yml`](../../docker-compose.yml), [`.env.example`](../../.env.example), [`models.yml`](../../models.yml), [`architecture.md`](../architecture.md), [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md), [`500k-context-2-parallel-agents-gpu-roadmap.md`](500k-context-2-parallel-agents-gpu-roadmap.md), [`qwen3.8-27b-tool-calling.md`](qwen3.8-27b-tool-calling.md)
+- llama.cpp: [LICENSE](https://github.com/ggml-org/llama.cpp/blob/master/LICENSE), [docs/speculative.md](https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md), [common/arg.cpp](https://github.com/ggml-org/llama.cpp/blob/master/common/arg.cpp), [common/speculative.cpp](https://github.com/ggml-org/llama.cpp/blob/master/common/speculative.cpp), PRs [#27342](https://github.com/ggml-org/llama.cpp/pull/27342), [#27624](https://github.com/ggml-org/llama.cpp/pull/27624), [#28058](https://github.com/ggml-org/llama.cpp/pull/28058), [#30997](https://github.com/ggml-org/llama.cpp/pull/30997), [#30498](https://github.com/ggml-org/llama.cpp/pull/30498), [#30567](https://github.com/ggml-org/llama.cpp/pull/30567), [#30574](https://github.com/ggml-org/llama.cpp/pull/30574), [#26004](https://github.com/ggml-org/llama.cpp/pull/26004), [#25051](https://github.com/ggml-org/llama.cpp/pull/25051); issues [#27148](https://github.com/ggml-org/llama.cpp/issues/27148), [#30961](https://github.com/ggml-org/llama.cpp/issues/30961), [#21681](https://github.com/ggml-org/llama.cpp/issues/21681), [#31472](https://github.com/ggml-org/llama.cpp/issues/31472), [#27117](https://github.com/ggml-org/llama.cpp/issues/27117), [#27833](https://github.com/ggml-org/llama.cpp/issues/27833), [#28777](https://github.com/ggml-org/llama.cpp/issues/28777), [#30075](https://github.com/ggml-org/llama.cpp/issues/30075), [#28867](https://github.com/ggml-org/llama.cpp/issues/28867), [#29820](https://github.com/ggml-org/llama.cpp/issues/29820), [#30884](https://github.com/ggml-org/llama.cpp/issues/30884)
+- vLLM: [GPU install docs](https://docs.vllm.ai/en/latest/getting_started/installation/gpu/) (page dated 2026-10-08), [Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base), [quantization README](https://github.com/vllm-project/vllm/blob/main/docs/features/quantization/README.md), [gguf.md](https://github.com/vllm-project/vllm/blob/main/docs/features/quantization/gguf.md), [config/cache.py](https://github.com/vllm-project/vllm/blob/main/vllm/config/cache.py), [config/speculative.py](https://github.com/vllm-project/vllm/blob/main/vllm/config/speculative.py), [LICENSE](https://github.com/vllm-project/vllm/blob/main/LICENSE), [Qwen3.8-27B recipe](https://recipes.vllm.ai/Qwen/Qwen3.8-27B), PRs/issues [#40977](https://github.com/vllm-project/vllm/pull/40977), [#64614](https://github.com/vllm-project/vllm/pull/64614), [#52637](https://github.com/vllm-project/vllm/pull/52637), [#57861](https://github.com/vllm-project/vllm/pull/57861), [#48723](https://github.com/vllm-project/vllm/issues/48723), [#65034](https://github.com/vllm-project/vllm/issues/65034), [ROCm/vllm#1229](https://github.com/ROCm/vllm/issues/1229)
+- vLLM RDNA4 fork: [repo](https://github.com/Capicua25x/vllm-rocm-rdna4), [RDNA4-PORT.md](https://github.com/Capicua25x/vllm-rocm-rdna4/blob/rdna4-port-0.28.0/RDNA4-PORT.md)
+- SGLang: [#30599](https://github.com/sgl-project/sglang/issues/30599), [#46900](https://github.com/sgl-project/sglang/issues/46900), [Qwen3.8-27B cookbook](https://docs.sglang.io/cookbook/autoregressive/Qwen/Qwen3.8-27B), [LICENSE](https://github.com/sgl-project/sglang/blob/main/LICENSE)
+- EAGLE-4: [repo README](https://github.com/eagle-hq/EAGLE-4), [LICENSE](https://github.com/eagle-hq/EAGLE-4/blob/main/LICENSE), [EAGLE-Corp/Qwen3.8-27B-EAGLE4](https://huggingface.co/EAGLE-Corp/Qwen3.8-27B-EAGLE4)
+- Weights: [Qwen/Qwen3.8-27B](https://huggingface.co/Qwen/Qwen3.8-27B) and [config.json](https://huggingface.co/Qwen/Qwen3.8-27B/raw/main/config.json), [Qwen/Qwen3.8-27B-FP8](https://huggingface.co/Qwen/Qwen3.8-27B-FP8), [unsloth GGUF](https://huggingface.co/unsloth/Qwen3.8-27B-GGUF), [z-lab DFlash2](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2), [z-lab DFlash2 GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF), [incoai DFlash2](https://huggingface.co/incoai/Qwen3.8-27B-DFlash2), [RedHatAI DSpark](https://huggingface.co/RedHatAI/Qwen3.8-27B-speculator.dspark), [RadixArk DSpark](https://huggingface.co/RadixArk/Qwen3.8-27B-DSpark), [RedHatAI INT4](https://huggingface.co/RedHatAI/Qwen3.8-27B-INT4), [AMD Quark INT4](https://huggingface.co/amd/Qwen3.8-27B-Quark-AWQ-INT4-W4A16)
+- Others: [TGI](https://github.com/huggingface/text-generation-inference), [ExLlamaV3](https://github.com/turboderp-org/exllamav3), [Lemonade](https://github.com/lemonade-sdk/lemonade), [Level1Techs vLLM ROCm regression thread](https://forum.level1techs.com/t/vllm-0-23-x-rocm-7-14-upgrade-on-4x-radeon-ai-pro-r9700-50-regression-on-one-model-unaffected-on-another-but-a-free-9-via-kernel-tuning/253460)
+
+Confidence: high for licenses, flag and spec-type lists, and issue/PR states (read via the GitHub and HF APIs on 2026-11-02). Medium for the third-party throughput figures (self-reported, not re-run). Low for the single-card vLLM VRAM estimate and the hybrid-state explanation of the typos, which are both marked as inference.

docs/research/qwen3.8-27b-decode-speed.md

@@ -0,0 +1,226 @@
+# Research: decode tokens/s on the R9700 (llama-server, Qwen3.8-27B + DFlash2)
+
+**Date:** 2026-10-20
+
+**Question:** Why is decode at ~52k context about plain-decode speed (26-28 t/s) when an older, shorter-context log averaged 45.9 t/s, and what can be changed to raise decode (and prefill where relevant)? Evidence: `llmserver.log` (re-analysed per request) plus llama.cpp source.
+
+**Scope note:** research only. Nothing in compose or `.env.example` was changed and nothing was deployed. This note builds on [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md) (cache/prefill ranking, DDR tier, five outside configs) and [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) (quant choice, VRAM budget). It does not repeat them, and it **corrects one premise of the first** (see finding 1).
+
+**Labels.** **[V]** verified from source (URL pinned to a commit, or a repo file:line). **[L]** computed from `llmserver.log` by me. **[R]** reported by a third party (a page or thread, not re-run). **[I]** inferred, not measured. WebFetch summaries of web pages were produced by a small model, so every number taken from a page is **[R]** even when the page is first-party.
+
+**Source pin.** llama.cpp `master` at `839a572a4cdecdb80d211297b8e7d7323a6e99d2` (fetched 2026-10-20). Links below point at that commit. The running image (`server-rocm`) may be older than this commit. Its build string is not in the logs I have, so every "[V]" is "true on that commit".
+
+## TL;DR
+
+| # | Finding | Label |
+|---|---|---|
+| 1 | The draft length has been **5, not 7**, in the old log too. All 306 `draft acceptance` summaries have `generated` divisible by 5 and `mean len = 1 + accepted / (generated / 5)`. So `n_max=5` in the new startup log is the same setting, not a new clamp. The earlier note's "n_max is 7" came from compose and was wrong | [L] + [V] |
+| 2 | master clamps `n_max` to `block_size - 1` (= 7 for block_size 8). 5 is below that, so no clamp fired. 5 must have been **requested**: most likely `LLAMA_SPEC_N_MAX=5` in the server's `.env` (the repo `.env.example` default differs, and `.env` is not in the repo) | [V] + [I] |
+| 3 | Decode speed tracks acceptance almost linearly (r = 0.97 over 256 requests) and acceptance falls with depth: 0.75 at <8k, 0.57 at 8-20k, 0.42 at 20-40k, 0.40 at 40-60k, while decode falls 64.6 to 31.9 t/s. The 45.9 t/s average was dominated by short-context requests | [L] |
+| 4 | Bandwidth ceiling for plain decode: 17.6 GB / 640 GB/s = 27.5 ms = **36.4 t/s** (33.0 t/s at 52k depth with q8_0 KV). At the 79-81 % efficiency best seen for dense models on this card that is 26-27 t/s, which is exactly the measured "plain" rate | [I] on [R] inputs |
+| 5 | A spec step costs 74-94 ms in the old log, against a ~28-30 ms weight-streaming floor. About 45 ms per step is fixed overhead (drafter, 6-wide verify, sync, sampling), so speculation only pays when acceptance is high | [L] + [I] |
+| 6 | DFlash2 draws its draft greedily and gives the server no draft distribution, so at temp > 0 a draft token is accepted only if the target's own sampled token equals it. Sampler settings on the client therefore change acceptance directly | [V] |
+| 7 | `GPU_MAX_HW_QUEUES=1` is free for default llama.cpp: multi-stream graph execution is opt-in (`GGML_CUDA_GRAPH_OPT=1`) | [V] |
+
+Top 3 experiments (details in "Experiment plan"): **A** sweep `--spec-draft-n-max` 3, 4, 5, 7 and add `--spec-draft-p-min`; **B** measure acceptance versus client sampler settings with a fixed prompt (temp, top-p/top-k, penalties); **C** decompose the 74-94 ms step with `llama-bench` (pp6 at depth vs tg1) and `rocm-smi` clocks, before touching KV type, slots or quant.
+
+## 1. DFlash2: why acceptance and speedup vary
+
+### 1.1 The n_max 5 vs 7 discrepancy
+
+- The init line prints `n_max=%d` from `this->params.n_max` **before** the clamp: [`common/speculative.cpp#L1086`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1086). The clamp is [`#L1090-L1099`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1090): `n_draft_max = (is_dspark && sample_from_anchor) ? block_size : block_size - 1`, with the warning `requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d`. **[V]**
+- For this drafter (`block_size=8`, DFlash2 is not DSpark) the limit is 7. So a requested 7 passes unclamped and a requested 5 prints 5. The init line therefore shows the **requested** value; no clamp explains 5. **[V]** (on that commit)
+- `n_extract=5` in the log is `target_layer_ids_n`, the number of target layers whose features feed the drafter ([`#L1040-L1044`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1040)). It is not used in the clamp, so the 5 = 5 match is a coincidence on this commit. **[V]** An older build could clamp differently; I could not check history for this. **[I]**
+- Log proof that 5 is the real draft length. The server prints `mean len = 1 + accepted / verif_steps` ([`tools/server/server-context.cpp#L704-L725`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L704)). With `verif_steps = generated / 5` this reproduces the printed mean len in all 306 summaries; with 7 it matches none. Examples: `llmserver.log:3` (47 accepted / 105 generated, 21 steps, 47/21 + 1 = 3.24), `:36` (64/170, 34 steps, 2.88), `:114` (828/2050, 410 steps, 3.02). **[L]**
+- To confirm on the server: `grep -n SPEC_N_MAX .env` and `docker logs llama-server | grep -E "n_max|exceeds the trained"`. **[I]**, not run.
+
+### 1.2 What the drafter does per step
+
+- The drafter input is `[id_last, <mask> * n_max]` and one forward pass denoises the whole block. DFlash2 then greedily walks a candidate lattice (`selector_top_k` candidates per position, argmax per step): [`#L1296-L1310`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1296) and [`#L1336-L1366`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1336). The block size in the drafter batch is `n_max + 1`, so a smaller `n_max` shrinks the drafter batch too. **[V]**
+- `--spec-draft-p-min` on DFlash2: stop extending the draft at the first position where softmax(selector scores) at the argmax is below `p_min` ([`#L1349-L1358`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1349)). It is a per-position stop, default 0.00 ([`common/common.h#L333`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L333), [`common/arg.cpp#L4248`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4248)). The git log shows DFlash2-specific p_min arrived with the DFlash2 PR ([#27342](https://github.com/ggml-org/llama.cpp/pull/27342), commit titles "Add p_min in DFlash2" per the PR page **[R]**). **[V]** for the code. The `generated` multiples of 5 in the log confirm p_min was 0 in the old run. **[L]**
+- `n_min` (`--spec-draft-n-min`, default 0): a draft shorter than `n_min` is discarded ([`#L1362-L1364`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1362)). **[V]**
+- `sample_from_anchor=true` only changes the layout for DSpark drafters; for DFlash2 the block is `n_max + 1` ([`#L1298`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1298)). **[V]**
+- `--spec-draft-sampling probabilistic` ([`arg.cpp#L4264`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4264)) is documented as sampling the draft and verifying by rejection. The DFlash2 branch never fills the draft distribution (`result_q`), and the server then falls back to plain sample-and-match: `use_rejection = use_spec_rejection() && !spec_draft_q.empty()` ([`server-context.cpp#L4537`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L4537)). So that flag does nothing for DFlash2. **[V]** (read of the code path; not run).
+
+### 1.3 Why acceptance varies
+
+1. **Sampling.** With the DFlash2 drafter, a draft token is accepted iff the target's sampled token equals it ([`common/sampling.cpp#L690-L720`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/sampling.cpp#L690), `draft[i] != id` breaks). At temp 0 the target token is its argmax, the best case for matching a greedy draft. At temp > 0 it falls with flatness of the target distribution and with anything that reshapes it (top-p/top-k, repetition or presence penalties). **[V]** for the mechanism. The size of the effect for this workload is unmeasured. **[I]**
+   - Contradiction to resolve: the earlier note cites a third-party claim that greedy sampling "collapses" DFlash2 acceptance. The source reading says the opposite for the server path. Treat that claim as unverified until experiment B settles it.
+   - Server-side the temp > 0 test is `task->params.sampling.temp > 0` ([`server-context.cpp#L508`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L508)). The drafter itself ignores temperature. **[V]**
+2. **Context depth.** In the old log, acceptance halves from <8k to >40k tokens (0.75 to 0.40, table below). A PR commenter also reported code prompts decaying faster with context length than prose ([PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) comments) **[R]**. The model card gives no context length for the drafter. **[R]**
+3. **Content type.** Not separable in this log (no prompt/response classes). The authors' acceptance lengths (mean tokens per verify step, 7 drafted, block size 8, temp 1.0 / top-p 0.95 / top-k 20, SGLang on an H200) are GSM8K 5.46, MATH-500 5.28, HumanEval 4.39, MBPP 4.79 ([z-lab/Qwen3.8-27B-DFlash2](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2), via a search summary) **[R]**. Your mean len of 2.9-4.9 (max possible 6 at n_max 5) is below that, partly because those are short prompts. **[I]**
+4. **Concurrency.** Issue [#27117](https://github.com/ggml-org/llama.cpp/issues/27117) (gfx1151, closed "not planned"): acceptance collapses as the number of concurrently drafting sequences grows (0.51-0.66 at 4 active, 0.06-0.49 at 16), and `--spec-draft-n-max 1` recovered it. **[R]** Your two slots are far below that regime, but the bimodal per-slot acceptance in that issue is worth watching if both slots decode at once. **[I]**
+
+### 1.4 Authors' recommended values
+
+- Model card quick start: `--spec-type draft-dflash --spec-draft-n-max 7` ([z-lab/Qwen3.8-27B-DFlash2-GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF)) **[R]**. No block size, temperature or p_min advice is given there. The card says the output is lossless (greedy identical, sampling distribution preserved).
+- A PR commenter on an RTX 3090 used `--spec-draft-n-max 4` and reported that 7 was slower: DFlash2 at n=4 averaged 62.21 t/s at 0.6153 acceptance, MTP 57.79 t/s at 0.6317 ([PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) comments) **[R]**.
+- Drafter quant does not matter for acceptance: acceptance length 5.28 (BF16), 5.13 (Q8_0), 5.39 (Q4_K_M) on 8 GSM8K prompts; the card itself implies Q4_K_M above BF16 is noise. Q4_K_M is ~1.1 GB, so it also costs the least bandwidth. **[R]**
+
+### 1.5 What the old log shows (re-analysis)
+
+Per-request join of `stop processing: n_tokens` (final depth), `eval time` t/s and `draft acceptance`, requests with >= 200 generated tokens, 256 requests (script: ad-hoc Python over `llmserver.log`, not committed). Draft length 5, so tokens per verify step is 1 + 5 x acceptance.
+
+| Final depth | n | Decode t/s (mean) | Acceptance (mean) | Tokens/step | Step time (tokens/step / t/s) |
+|---|---|---|---|---|---|
+| < 8k | 125 | 64.6 | 0.75 | 4.75 | ~74 ms |
+| 8-20k | 28 | 44.2 | 0.57 | 3.85 | ~87 ms |
+| 20-40k | 17 | 34.6 | 0.42 | 3.10 | ~90 ms |
+| 40-60k | 86 | 31.9 | 0.40 | 3.00 | ~94 ms |
+
+- Correlation of per-request decode t/s with acceptance: r = 0.97, slope ~90 t/s per 1.0 acceptance. **[L]**
+- Step time rises only ~20 ms from 0 to 50k depth, while tokens per step drops 4.75 to 3.0. So about **80 % of the decode loss with depth is acceptance, not attention cost**. **[L]** (derived, mean-of-ratios caveat).
+- The new run (26-28 t/s at ~52k, tg_3s 20-33) is at or just below the 40-60k bucket (31.9), consistent with the same acceptance regime. **[L]**
+
+## 2. Decode speed versus context depth
+
+### 2.1 What the attention costs here
+
+- Only 16 of 64 layers keep a KV cache ([`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) "Model identity"). KV bytes per token: 16 layers x 2 x 4 heads x 256 x element size = 65,536 B (f16), 34,816 B (q8_0, 1.0625 B/element), 18,432 B (q4_0, 0.5625 B/element). At 52k tokens: 3.4 GB (f16), 1.81 GB (q8_0), 0.96 GB (q4_0). Against 17.6 GB of weights read per token, q8_0 KV adds ~10 % bytes at 52k, f16 ~19 %, q4_0 ~5 %. **[I]** (arithmetic from the quant note's architecture facts.)
+- So KV precision can move plain decode at 52k by at most a few percent. Going to q4_0 would buy ~1 GB less read (~5 %) at an unmeasured quality cost. Do not expect it to fix the 2x gap. **[I]**
+
+### 2.2 Which flash-attention kernel runs on gfx1201
+
+From [`ggml/src/ggml-cuda/fattn.cu#L540-L722`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L540) (`ggml_cuda_get_best_fattn_kernel`), applied to head_dim 256, 24 query heads, 4 KV heads (gqa_ratio 6, effective ratio 2), K/V q8_0:
+
+| Query batch (tokens) | Kernel on gfx1201 | Why (source) |
+|---|---|---|
+| 1 (plain decode) | VEC | WMMA needs `Q->ne[1] * gqa_ratio_eff > 16` for head_dim > 128 ([`#L698-L702`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L698)), so 1 x 2 fails. For quantized K/V the vector kernel is chosen when `Q->ne[1] <= 2` ([`#L714-L720`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L714)). Reads q8_0 directly. |
+| 2 | VEC (quantized) | same rule |
+| 3 to 8 (verify batch of a spec step) | **TILE** | 8 x 2 = 16 is not > 16, so no WMMA; quantized KV is not eligible for VEC above 2 tokens. |
+
+- TILE and MMA kernels "need f16 K/V" ([`#L735-L740`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L735)): with a quantized cache the K and V views are converted to f16 scratch before the kernel runs. So a verify step at 52k converts ~100M q8_0 elements per layer (K plus V) to f16 on 16 layers. Rough extra traffic: read 1.8 GB plus write 3.4 GB plus read 3.4 GB per step, ~8.6 GB, ~13 ms at 640 GB/s. **[I]** (the conversion and kernel choice are [V]; the traffic estimate and its cost are inferred, not profiled). That is consistent with the ~20 ms step-time rise from 0 to 50k in 1.5. With f16 KV the conversion disappears (TILE reads f16 directly) but the cache is 1.9x larger, so the net is not obvious. **[I]**
+- No kernel-level bug report for gfx1201 + quantized KV turned up in the sources I read. The only RDNA4 data point is [R]: on RADV (Vulkan) q8_0 vs q4_0 KV was within +-0.5 % ([discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043)). That is a different backend and says nothing about the HIP path.
+
+### 2.3 `--parallel 2` and `--kv-unified`
+
+- With `--kv-unified` there is one KV stream. `get_n_kv` pads `cells.used_max_p1()`, the highest occupied cell across **all** sequences ([`src/llama-kv-cache.cpp#L1282-L1292`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-kv-cache.cpp#L1282)). The flash-attention kernels only skip fully masked tiles at the **end** of the range (`flash_attn_mask_to_KV_max`, [`fattn-common.cuh#L671-L723`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn-common.cuh#L671)). So when the other slot holds a long prompt in higher cells, slot 0's decode may read and mask-skip past tens of thousands of foreign cells. **[V]** for both pieces, **[I]** for the combined cost (not profiled; which cells each slot occupies is not logged).
+- This matches the pattern in the older log (requests alternate slot 0 / slot 1, each ~40-50k tokens, [throughput note](qwen3.8-27b-throughput-tuning.md) "Pattern at `:380`-`:470`"), but no measurement separates it. **Experiment D** below tests it.
+- `--kv-unified-per-slot N` exists ([`arg.cpp#L1652`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L1652)) and `--no-kv-unified` gives one stream per slot ([`arg.cpp#L1727-L1734`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L1727)). Non-unified mode splits `--ctx-size` evenly, so one slot could no longer use more than half the context. **[V]**
+
+### 2.4 Checkpoints and recurrent state during decode
+
+- The target is `qwen35` architecture, which `llm_arch_supports_rs_rollback` lists ([`src/llama-arch.cpp#L1256-L1262`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-arch.cpp#L1256)). I am assuming Qwen3.8 loads as that arch; the name was not verified. **[I]**
+- For DFlash/DSpark the server requests `n_rs_seq = draft.n_max` recurrent-state snapshots per sequence ([`common/common.h#L401-L407`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L401); [`src/llama-context.cpp#L191-L195`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-context.cpp#L191)). The server creates a **speculative checkpoint** only if `draft.size() > llama_n_rs_seq(ctx_tgt)` ([`server-context.cpp#L3541-L3558`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L3541)), and restores one only if `n_rollback > n_rs_seq` ([`#L4559`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L4559)). With `n_rs_seq == n_max` that never happens, so a rejection is a snapshot-index rollback and not a state copy. **[V]**
+- If the build is older or the arch is not on the rollback list, the `COMMON_CONTEXT_SEQ_RM_TYPE_FULL` path copies a checkpoint every step (`use_ckpt_tgt`, [`#L3463`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L3463)); the startup line `speculative decoding will use checkpoints` ([`#L1388-L1390`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L1388)) would show it. Check the startup log for that line. **[V]** for the code, **[I]** for which case applies.
+- `--ctx-checkpoints` / `--checkpoint-min-step` (defaults 32 and 8192, [`common.h#L639-L642`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L639)) are for prompt/prefix reuse. Nothing I read ties them to the decode loop, so they should not cost decode time. **[I]**
+- The "cache_reuse is not supported by this context" startup line is the server's `llama_memory_can_shift` check ([`server-context.cpp#L1328-L1336`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L1328)). It confirms the open question in the throughput note. **[V]**
+
+## 3. Backend, HIP flags, clocks
+
+### 3.1 ROCm versus Vulkan (RADV)
+
+Everything here is **[R]** unless noted; the thread posts are self-reported and mixed in setup.
+
+| Source | Setup | Numbers |
+|---|---|---|
+| [discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043) (JohnTDI-cpu, build 8554) | R9700, RADV, Qwen3.5-27B Q4_K_M | tg128 29.07 t/s (RADV), 29.01 (AMDVLK); pp512 798 (RADV). Computed there as ~456 GB/s, 71 % of 640 GB/s; "best dense ~79-81 %" |
+| same | R9700, ROCm, 27B Q8_0, q4_0 KV, 64k ctx | pp 960 (3.7k) to 708 (58.7k) t/s; tg 42.8 to 28.0 t/s over the same range (MTP use not stated, and 42.8 t/s is above a 29 GiB Q8_0 plain-decode ceiling of ~22 t/s, so it is almost certainly with speculation) |
+| same | Windows, HIP, gfx1201 | pp512 1106-1211; tg128 29.07; "Vulkan decode still wins" |
+| same | R9700, Qwen3.6-27B UD-Q4_K_XL, q4_0 KV, MTP (build b9203) | tg128 28.2 baseline, 47.4 with `mtp-3` |
+| [discussion #20881](https://github.com/ggml-org/llama.cpp/discussions/20881) | R9700 and others | MoE Qwen3-Coder-30B-A3B Q4_0 ~196 t/s on RADV; one ROCm segfault report on Qwen3.5-35B-A3B |
+| [throughput note](qwen3.8-27b-throughput-tuning.md) (cat5edopeHA) | R9700, MTP n3 | Vulkan 68.5 / ROCm 55.1 short, 50.5 / 40.2 at 32k, 40.9 / 31.2 at 64k |
+
+- A search summary also cites a blog (not fetched) with ROCm ahead for dense 27B decode (42.8 vs 29.1), the opposite of cat5edopeHA. The sources conflict, so only a local A/B decides. **[R]**
+- DFlash2 on Vulkan: not verified. The DFlash graph and feature extraction live in common code and the model graph, so it may work, but I did not check that a Vulkan build runs `draft-dflash`. **[I]**
+- Prefill: HIP uses MMQ on RDNA4 unconditionally (`ggml_cuda_should_use_mmq` returns true for RDNA4, [`mmq.cu#L468-L471`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmq.cu#L468)), so `GGML_CUDA_FORCE_MMQ` changes nothing on gfx1201. **[V]**
+
+### 3.2 Decode kernels for the verify batch
+
+- Weights are Q4_K-family. Matrix multiplies with up to 8 columns go to MMVQ (`MMVQ_MAX_BATCH_SIZE 8`, [`mmvq.cuh`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmvq.cuh)). RDNA4 has no per-type override, so `ne11 <= 8` uses MMVQ ([`mmvq.cu#L318-L430`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmvq.cu#L318)). The source comment says k-quants "cost more to decode and mvq redoes that per column". **[V]** So the verify batch of 6 (n_max 5) or 8 (n_max 7) pays dequant ALU work per column; cost grows with batch width even though weight bytes are read once. **[I]** (the per-column cost on gfx1201 was not measured). This is the mechanism behind "lower n_max can win": it shrinks both the MMVQ columns and the drafter batch.
+- Batch 9 or more would fall to MMQ, which is not reachable: `n_max` is capped at 7 so the batch is at most 8. **[V]**
+
+### 3.3 HIP flags and environment variables
+
+| Item | Verdict | Source |
+|---|---|---|
+| `GGML_HIP_GRAPHS` | Already ON in the CMake default and the official ROCm Dockerfile does not turn it off. Variable graph shapes force a re-warmup: any change in node properties resets `warmup_complete` ([`ggml-cuda.cu#L4806-L4845`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/ggml-cuda.cu#L4806)). With `p_min > 0` the verify batch width varies, so graphs may be re-captured more often. Debug with `GGML_CUDA_GRAPH_DEBUG=1`; disable with `GGML_CUDA_DISABLE_GRAPHS=1`. | [V] ([`ggml/CMakeLists.txt#L217`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/CMakeLists.txt#L217), [`rocm.Dockerfile`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/.devops/rocm.Dockerfile)); effect on speed **[I]** |
+| `GGML_CUDA_FORCE_MMQ` | No effect on RDNA4 (see 3.1) | [V] |
+| `GGML_CUDA_GRAPH_OPT=1` | Opt-in multi-stream concurrent branches ([`ggml-cuda.cu#L5016-L5030`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/ggml-cuda.cu#L5016)). Off by default | [V] |
+| `GPU_MAX_HW_QUEUES=1` | HIP caps hardware queues per process per device (default 4, [HIP env docs](https://rocm.docs.amd.com/projects/HIP/en/docs-7.2.3/reference/env_variables.html)). llama.cpp issues all kernels on one stream unless `GGML_CUDA_GRAPH_OPT=1`, so one queue should cost nothing for decode. The repo set it for [ROCm/ROCm#5706](https://github.com/ROCm/ROCm/issues/5706) (`docs/architecture.md#gpu-access`). Do not combine `GGML_CUDA_GRAPH_OPT=1` with it without a test | [R] for the doc, [V] for the llama.cpp side, **[I]** for "no decode cost" |
+| `-bs, --backend-sampling` | Experimental, default off ([`arg.cpp#L2310`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L2310)). Moves target sampling onto the GPU, which would remove the CPU sampling of 6 logit rows per step. Interplay with speculation is unverified | [V] flag exists; benefit **[I]** |
+| `rocm-smi` perf level / compute profile | In #21043: "ROCm `rocm-smi` high vs auto: auto was faster" on a single card; a LACT 330 W cap with COMPUTE profile gave tg 102 to 110 on a dual-card MoE run; `power_dpm_force_performance_level=high` gave +14 % tg on one Vulkan run. Mixed, hardware and backend differ | [R] |
+| PCIe ASPM `pcie_aspm.policy=performance` | Same thread: dense decode on RADV 29.30 to 32.46 (+10.8 %), one user, not reproduced on eGPU or dual-slot. It is a runtime sysfs switch, so it is cheap to try | [R] |
+| MCLK not reaching max under load | Same thread, kernel 6.17, RADV | [R]; check with `rocm-smi` during decode, **[I]** |
+
+### 3.4 Threads, mmap, mlock, poll
+
+- With every layer on the GPU the CPU is only a scheduler. #21043 tested `-t`, `-b`, `-mmp 0`, `--prio 2`, CPU pinning, hugepages and `nice` on RADV and found all within +-0.3 % **[R]**. The server default `--poll` level is 50 ([`common.h#L77`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L77)) and `-t` was -1 (16 here). I found no evidence these matter for a GPU-resident run, so do not spend experiments on them. **[I]**
+- One exception worth knowing: speculative drafting has its own thread/poll flags (`--spec-draft-threads`, `--spec-draft-poll`, [`arg.cpp`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp)), but the drafter is on the GPU too, so they should be equally inert. **[I]**
+
+## 4. Quant and model options
+
+Inputs: GGUF sizes from [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) (UD-Q4_K_XL 17.6 GB, UD-Q4_K_M 16.5, UD-Q4_K_S 15.4) and the throughput note (UD-IQ4_XS 13.3 GB). Bandwidth 640 GB/s is the card spec as quoted in #21043 **[R]**. Weights only; 52k q8_0 KV adds 1.81 GB (2.1).
+
+| Quant | Bytes read/token | Ceiling at 640 GB/s | At 80 % efficiency | Same with 52k q8_0 KV (ceiling / 80 %) |
+|---|---|---|---|---|
+| UD-Q4_K_XL (running) | 17.6 GB | 36.4 t/s (27.5 ms) | 29.1 | 33.0 / 26.4 |
+| UD-Q4_K_M | 16.5 GB | 38.8 | 31.0 | 35.0 / 28.0 |
+| UD-Q4_K_S | 15.4 GB | 41.6 | 33.2 | 37.2 / 29.8 |
+| UD-IQ4_XS | 13.3 GB | 48.1 | 38.5 | 43.5 / 34.8 |
+
+- All **[I]** (arithmetic). The 80 % factor is the "best dense ~79-81 %" figure in #21043 **[R]**. The measured plain-decode rate at depth, 26-28 t/s, sits on the 52k column for Q4_K_XL (26.4), so the card is already near its bandwidth limit for plain decode. There is no hidden 2x in the non-speculative path.
+- A smaller quant gains at most ~13 % (Q4_K_M) to ~32 % (IQ4_XS) on the plain-decode ceiling and also shrinks the spec verify weight read. Quality cost is not measured here. The IQ4 family dequantizes slower than Q4_K, so the real gain is below the ceiling ratio. **[I]** Test with KL divergence or perplexity on a small fixed corpus (`llama-perplexity`) before any switch.
+- Drafter quant: Q4_K_M at ~1.1 GB is already the smallest tested, with acceptance as good as BF16 (1.4). Nothing to gain. **[R]**
+- Per spec step the drafter adds ~1.1 GB of reads (~1.7 ms at the ceiling), small against the ~45 ms fixed overhead found in 1.5. **[I]**
+
+## 5. Where the step time goes (open)
+
+Old-log step times are 74 ms at <8k and 94 ms at 40-60k ([1.5](#15-what-the-old-log-shows-re-analysis)). The weight-streaming floor at 80 % efficiency is ~34 ms. The remaining ~40-60 ms per step is not explained by any single item I could verify. Candidate contributors, none measured:
+
+1. 6-column MMVQ with k-quant dequant per column (3.2).
+2. Drafter forward over `n_max + 1` positions plus injecting 5 layers of target features per accepted token (`n_extract=5`, [`speculative.cpp#L1040`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1040)), including the encoder fused into KV injection ([commit 662a0b0](https://github.com/ggml-org/llama.cpp/commit/662a0b012)).
+3. CPU sampling of 6 full-vocab logit rows plus device-to-host copies per step (`-bs` is off).
+4. Host synchronisation between drafter and target per step (two contexts, no overlap).
+5. Quantized-to-f16 KV conversion for the TILE kernel at depth (2.2).
+
+Experiment C isolates 1 and 5 without the server; the rest need a profile (`rocprofv3`), which is out of scope here.
+
+## 6. Experiment plan
+
+Rules: one change per run, cold slot, same prompt, restart the server between configs with `./scripts/update.sh` on the server (this box cannot deploy). All numbers come from the server log lines `prompt eval time`, `eval time` and `draft acceptance` (`llmserver.log:32-36` show the format).
+
+### Benchmark protocol (minimal, comparable)
+
+1. **Fixed prompts** (kept in the repo's scratch area, not committed): P-code = ~8k tokens of source plus the instruction "continue the file" and P-code-52k = the same material padded to ~52k tokens; P-reason = a reasoning question over the same 52k context. Same bytes every run.
+2. **Fixed request:** `n_predict=512` (400+ tokens so the 3-s `tg` window averages out), `seed` fixed, `cache_prompt` false for the first request so prefill is cold, then the same request again with the prompt cache warm only if you want decode in isolation.
+3. **Sampler:** run each prompt twice, once with the production client settings (record them from the request body) and once with `temperature=0`. Record the sampler fields in the run name.
+4. **Repeats:** N = 5 per cell, both slots idle except the request under test; discard the first.
+5. **Read** from the log for each request: prompt tokens/s (`prompt eval time`), decode t/s (`eval time`), `draft acceptance` and `mean len`. Report median and min/max, not the mean.
+6. **Decode-only control:** `llama-bench` inside a throwaway container (stop `llama-server` first, one GPU): `llama-bench -m <model> -ngl 99 -fa 1 -ctk q8_0 -ctv q8_0 -p 0 -n 256 -d 0,16384,52000 -r 3`. `-n 256` avoids the "implicit ubatch 128 on hybrid models" artifact reported with `-n 128` in #21043 **[R]**. `llama-bench` does not use speculation or the prompt cache.
+7. **Environment capture** per run: `rocm-smi --showclocks --showpower --showtemp` once during the decode (SCLK, MCLK), plus the full startup log (build string, `n_rs_seq`, checkpoint line).
+
+### Experiments, in priority order
+
+| # | Change (exact) | Expected effect | How to measure | Risk |
+|---|---|---|---|---|
+| A | `LLAMA_SPEC_N_MAX` = 3, 4, 5, 7 (one run each); then 5 with `--spec-draft-p-min 0.3` and `0.5` | Fewer wasted drafted tokens and a narrower verify batch when acceptance is ~0.4; could be neutral or negative when acceptance is high. Direction at 52k is the point; short context favors the larger value. No estimate of size without data | Decode t/s, `mean len`, `generated` (must be a multiple of the n_max, or not if p_min is on), on P-code and P-code-52k | Low. `p_min` makes batch widths vary, which can trigger graph re-capture (3.3); if so, compare with `GGML_CUDA_DISABLE_GRAPHS=1` |
+| B | No server change: send P-code-52k / P-reason with (1) production sampler, (2) `temperature=0`, (3) production minus penalties, (4) production minus top-p/top-k | Isolates how much acceptance the sampler costs (6, 1.3). A large gap says the fix is in the client/gateway sampler defaults, not llama.cpp | `draft acceptance` and `mean len` per cell; the full per-position `acc per pos` appears at trace level only | None |
+| C | Stop `llama-server`; in a container with the same image: `llama-bench ... -p 1,2,4,6,8 -n 0 -d 0,52000` and `-p 0 -n 256 -d 0,52000`; rocm-smi clocks during a server decode | Splits step time into verify-batch cost (pp6 time at depth) vs plain tg1. Shows whether verify at depth already costs ~60+ ms | `llama-bench` t/s table; compare pp6 time with measured step time | Low. Needs a short outage of the only GPU consumer |
+| D | `--parallel 1` (then `--no-kv-unified` with `--parallel 2`) on the same P-code-52k | Tells whether the second slot or the unified cache costs single-stream decode (2.3). If decode at 52k jumps, slot layout is the cost | Decode t/s on a request with the other slot idle; then with the other slot holding a 50k context | Medium. `--parallel 1` serialises agents; non-unified halves per-slot max context |
+| E | `echo performance \| sudo tee /sys/module/pcie_aspm/parameters/policy`, and check MCLK/SCLK under load; try `rocm-smi --setperflevel high` only if MCLK sags | Reported +10.8 % on one RADV dense run; unknown for ROCm. Free to try and revert | Decode t/s on P-code (plain control from C) | Low; revert with the previous policy value |
+| F | `-bs` (`--backend-sampling`) | Removes CPU sampling of the verify rows if that is a real share of the step | A/B on P-code | Medium: experimental, speculation interplay unverified |
+| G | `GGML_CUDA_DISABLE_GRAPHS=1` and, separately, `GGML_CUDA_GRAPH_DEBUG=1` to see re-warmup counts | Tells whether graph churn from the variable shapes is a cost | A/B on P-code | Low |
+| H | `-ctk q4_0 -ctv q4_0` at 52k | Saves ~0.85 GB read per token (~4-5 % on plain decode) and removes some f16 conversion traffic; quality risk | A/B on P-code-52k plus a spot-check of output quality | Medium (quality) |
+| I | `-ctk f16 -ctv f16 --ctx-size 131072` | Removes q8_0 to f16 conversion for the TILE verify kernel (2.2); cache is 1.9x larger. VRAM: f16 KV for 131072 tokens is ~8 GB | A/B on P-code-52k | Medium (halves context, VRAM) |
+| J | Smaller quant (UD-Q4_K_M or UD-IQ4_XS) from the same repo | Up to +13 % / +32 % on the plain ceiling (4); quality unmeasured | A/B plus KLD/perplexity | Medium (quality, download) |
+| K | Vulkan (RADV) image, if `draft-dflash` runs on it | Reported decode lead on some setups, but conflicting sources and prefill lag at short context | A/B with the same protocol | High (new image, unverified DFlash2 support) |
+| L | `--spec-type draft-dflash,ngram-mod` (list is accepted, [`arg.cpp#L4314-L4322`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4314)) | Extra drafts on repeated text (tool JSON, code). Interaction with the DFlash drafter is unchecked | A/B on P-code | Medium |
+
+Not worth an experiment: `-t`, `--poll`, `--no-mmap`, `--mlock`, `GGML_CUDA_FORCE_MMQ`, drafter quant (3.4, 3.1, 1.4).
+
+## Open questions
+
+- What does the server `.env` say for `LLAMA_SPEC_N_MAX`, and what does the new startup log print for `speculative decoding will use checkpoints` and `n_rs_seq`?
+- What temperature, top-p/top-k and penalties do OmniRoute clients actually send? If they differ per client, acceptance will differ per client.
+- Is the ~45 ms fixed overhead per spec step in the target verify, the drafter, or the CPU? Experiment C and a `rocprofv3` trace would answer it.
+- Does a Vulkan build run `draft-dflash` at all?
+- Which exact llama.cpp build is in the image, and is it before or after the `n_rs_seq` snapshot change?
+
+## Sources
+
+- Repo: `llmserver.log` (lines cited inline), `docker-compose.yml`, `.env.example`, `docs/architecture.md#llama-server`, `docs/research/qwen3.8-27b-throughput-tuning.md`, `docs/research/qwen3.8-27b-quant.md`.
+- llama.cpp at `839a572a4cdecdb80d211297b8e7d7323a6e99d2`: `common/speculative.cpp`, `common/arg.cpp`, `common/common.h`, `common/sampling.cpp`, `tools/server/server-context.cpp`, `src/llama-kv-cache.cpp`, `src/llama-arch.cpp`, `src/llama-context.cpp`, `src/llama-memory-recurrent.cpp`, `ggml/src/ggml-cuda/{fattn.cu,fattn-common.cuh,mmq.cu,mmvq.cu,mmvq.cuh,ggml-cuda.cu}`, `ggml/CMakeLists.txt`, `.devops/rocm.Dockerfile`. Line numbers were read from `raw.githubusercontent.com/.../master/...` on the day of fetch; if the link anchors drift, search by the quoted identifier.
+- [PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) (DFlash2), [issue #27117](https://github.com/ggml-org/llama.cpp/issues/27117), [discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043), [discussion #20881](https://github.com/ggml-org/llama.cpp/discussions/20881).
+- [z-lab/Qwen3.8-27B-DFlash2-GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF), [z-lab/Qwen3.8-27B-DFlash2](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2) (via a search summary), [HIP environment variables](https://rocm.docs.amd.com/projects/HIP/en/docs-7.2.3/reference/env_variables.html).
+- Not fetched: the Phoronix R9700 ROCm vs Vulkan review (HTTP 403), AMD ROCm RDNA4 pages beyond the HIP env page.

models.yml

@@ -0,0 +1,7 @@
+# Model files the downloader fetches into the models volume; see docs/operations.md#model-downloads
+# `file` must match the name in .env that loads it (LLAMA_MODEL_FILE, or -md in LLAMA_SPEC_ARGS).
+models:
+  - file: Qwen3.8-27B-UD-Q4_K_XL.gguf
+    url: https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-UD-Q4_K_XL.gguf
+  - file: Qwen3.8-27B-DFlash2-Q8_0.gguf
+    url: https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF/resolve/main/Qwen3.8-27B-DFlash2-Q8_0.gguf

scripts/bench.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Benchmarks llama-server with SPEED-Bench; see docs/operations.md#speed-bench
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+usage() {
+  echo "Usage: $0 run <name> [speed_bench args...]   # writes bench-results/<name>.json" >&2
+  echo "       $0 compare <baseline> <speculative>   # names from earlier runs" >&2
+  exit 1
+}
+
+case "${1:-}" in
+  run)
+    [ $# -ge 2 ] || usage
+    name=$2; shift 2
+    mkdir -p bench-results
+    docker compose --profile tools run --rm speed-bench --output "$name.json" "$@"
+    ;;
+  compare)
+    [ $# -eq 3 ] || usage
+    docker compose --profile tools run --rm --entrypoint python speed-bench \
+      /app/speed_bench_compare.py --baseline "$2.json" --speculative "$3.json"
+    ;;
+  *) usage ;;
+esac

scripts/redownload-model.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Forces a fresh download of a model file listed in models.yml, replacing the one on disk.
+# Why and how: docs/operations.md#forcing-a-model-re-download
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [ $# -gt 1 ] || [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
+  echo "Usage: $0 [model-file]   (default: LLAMA_MODEL_FILE from .env)" >&2
+  exit 1
+fi
+
+file="${1:-}"
+if [ -z "$file" ]; then
+  [ -f .env ] || { echo ".env missing, run scripts/update.sh first" >&2; exit 1; }
+  file="$(grep -m1 '^LLAMA_MODEL_FILE=' .env | cut -d= -f2- | tr -d '\r"'"'" || true)"
+  [ -n "$file" ] || { echo "LLAMA_MODEL_FILE is not set in .env" >&2; exit 1; }
+fi
+grep -qxF "  - file: $file" models.yml || { echo "$file is not listed in models.yml" >&2; exit 1; }
+
+# Runs a snippet against the models volume; $1 in it is the model file.
+in_volume() { docker compose --profile tools run --rm --entrypoint sh downloader -c "cd /models; $1" _ "$file"; }
+
+# Put the previous file back if the download didn't finish, so a failed run never loses the working model.
+ok=false
+restore() {
+  $ok || in_volume '[ ! -f "$1.old" ] || { rm -f "$1" "$1.part"; mv "$1.old" "$1"; }' || true
+}
+trap restore EXIT
+
+echo "==> moving aside the current $file (kept until the new download succeeds)"
+in_volume '[ -f "$1" ] || { [ ! -f "$1.old" ] || mv "$1.old" "$1"; }; [ ! -f "$1" ] || mv -f "$1" "$1.old"; rm -f "$1.part"'
+
+echo "==> downloading $file"
+docker compose --profile tools run --rm downloader
+
+ok=true
+in_volume 'rm -f "$1.old"'
+
+echo "==> restarting llama-server to load the new file"
+docker compose restart llama-server

scripts/update.sh

@@ -175,7 +175,6 @@ docker compose build --pull
 
 echo "==> ensuring models are downloaded (skips already-present files)"
 docker compose --profile tools run --rm downloader
-docker compose --profile tools run --rm downloader-draft
 
 echo "==> bringing up omniroute"
 docker compose up -d --wait omniroute

speed-bench/Dockerfile

@@ -0,0 +1,9 @@
+# see docs/operations.md#speed-bench
+FROM python:3.12-slim
+ARG LLAMA_CPP_REF=0ec4027a2a0183de87c007069adddc06e665f7de
+ADD https://raw.githubusercontent.com/ggml-org/llama.cpp/${LLAMA_CPP_REF}/tools/server/bench/speed-bench/requirements.txt \
+    https://raw.githubusercontent.com/ggml-org/llama.cpp/${LLAMA_CPP_REF}/tools/server/bench/speed-bench/speed_bench.py \
+    https://raw.githubusercontent.com/ggml-org/llama.cpp/${LLAMA_CPP_REF}/tools/server/bench/speed-bench/speed_bench_compare.py \
+    /app/
+RUN pip install --no-cache-dir -r /app/requirements.txt
+WORKDIR /results