Merge pull request 'docs(research): add qwen3.8-27b decode speed analysis' (#68) from docs/decode-speed-research into main

Reviewed-on: #68

.env.example

@@ -5,11 +5,11 @@ LLAMA_MODEL_FILE=Qwen3.8-27B-UD-Q4_K_XL.gguf
 LLAMA_GPU_LAYERS=999
 LLAMA_CTX_SIZE=262144
 LLAMA_PARALLEL=2
-LLAMA_CACHE_RAM=0
+LLAMA_CACHE_RAM=8192
 LLAMA_KV_CACHE_TYPE=q8_0
 
 # Blank = speculative decoding off; see docs/architecture.md#llama-server
-LLAMA_SPEC_ARGS=--spec-type draft-dflash -md /models/Qwen3.8-27B-DFlash2-Q8_0.gguf --spec-draft-n-max 7
+LLAMA_SPEC_ARGS=--spec-type draft-dflash -md /models/Qwen3.8-27B-DFlash2-Q8_0.gguf --spec-draft-n-max 3 --spec-draft-p-min 0.5
 
 # --- SearXNG web search --- see docs/gateway.md#web-search
 # Auto-filled by update.sh — leave blank.

.serena/.gitignore

@@ -0,0 +1,2 @@
+/cache
+/project.local.yml

.serena/project.yml

@@ -0,0 +1,187 @@
+# the name by which the project can be referenced within Serena/when chatting with the LLM.
+project_name: "LLM-Server"
+
+# list of language servers to start when using the LSP backend; choose from:
+#   ada                    al                     angular                ansible                astro
+#   bash                   bsl                    clojure                cpp                    cpp_ccls
+#   crystal                csharp                 csharp_omnisharp       cue                    dart
+#   deno                   elixir                 elm                    erlang                 fortran
+#   fsharp                 gdscript               gleam                  go                     groovy
+#   haskell                haxe                   hlsl                   html                   java
+#   json                   julia                  julia_fatou            kotlin                 latex
+#   lean4                  lua                    lua_emmylua            luau                   markdown
+#   matlab                 msl                    nextflow               nix                    ocaml
+#   pascal                 perl                   php                    php_devsense           php_phpactor
+#   php_phpantom           powershell             python                 python_basedpyright    python_jedi
+#   python_pyrefly         python_ty              qml                    r                      rego
+#   ruby                   ruby_solargraph        rust                   scala                  scss
+#   solidity               svelte                 swift                  systemverilog          terraform
+#   toml                   typescript             typescript_tsgo        typescript_vts         vue
+#   vbnet                  wolfram                yaml                   zig
+#   (This list may be outdated; generated with scripts/print_language_list.py;
+#   For the current list, see values of the LanguageServerId enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
+# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Astro projects, use astro (subsumes typescript/javascript for .astro projects; requires npm)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+#   - External Python adapters may add further registered IDs; install the adapter package first
+#     and then use its ID here, for example: example
+# Special requirements:
+#   Some language servers require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple language servers, the first language server that supports a given file will be used for that file.
+# The first language server is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+language_servers:
+- bash
+
+# the encoding used by text files in the project
+# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
+encoding: "utf-8"
+
+# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
+# the command runs in the project root directory and is only executed if the project is trusted
+# (see trusted_project_path_patterns in the global configuration).
+# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
+# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
+# backstop for non-terminating commands; on expiry the process is killed and activation continues.
+# example: activation_command: "npx nx run-many -t build"
+activation_command:
+
+# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
+# must be a positive number.
+activation_command_timeout: 180.0
+
+# line ending convention to use when writing source files.
+# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
+# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
+line_ending:
+
+# The language backend to use for this project.
+# If not set, the global setting from serena_config.yml is used.
+# Valid values: LSP, JetBrains
+# Note: the backend is fixed at startup. If a project with a different backend
+# is activated post-init, an error will be returned.
+language_backend:
+
+# The interface through which the agent (LLM) accesses Serena's functionality (overrides the global setting).
+# Valid values: tools, REPL (see the global configuration for details); leave empty to use the global setting.
+# Note: the interface is fixed at startup. If a project is activated post-init, its setting is not applied.
+agent_interface:
+
+# whether to use project's .gitignore files to ignore files
+ignore_all_files_in_gitignore: true
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
+ls_specific_settings: {}
+
+# list of workspace folder paths (LSP backend only).
+# These folders will be used to build up Serena's symbol index.
+# Paths must be within the project root and should thus be relative to the project root.
+# Furthermore, the paths should not be filtered by ignore settings.
+# Default setting: The entire project root folder (".") is considered.
+# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
+#   ls_workspace_folders:
+#     - "./subproject1"
+#     - "./subproject2"
+ls_workspace_folders:
+- "."
+
+# list of additional workspace folder paths for cross-package reference support.
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries, but these folders are not indexed by Serena,
+# i.e. the respective symbols will not be found using Serena's symbol search tools.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+ls_additional_workspace_folders: []
+
+# list of additional paths to ignore in this project.
+# Same syntax as gitignore, so you can use * and **.
+# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
+# Example:
+#   ignored_paths:
+#     - "examples/**"
+#     - ".worktrees/**"
+#     - "**/bin/**"
+#     - "**/obj/**"
+# Note: global ignored_paths from serena_config.yml are also applied additively.
+ignored_paths: []
+
+# whether the project is in read-only mode
+# If set to true, all editing tools will be disabled and attempts to use them will result in an error
+# Added on 2025-04-18
+read_only: false
+
+# list of tool names to exclude.
+# This extends the existing exclusions (e.g. from the global configuration)
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+excluded_tools: []
+
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+fixed_tools: []
+
+# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.find_symbol") to exclude from the REPL.
+# This extends the existing exclusions (e.g. from the global configuration).
+excluded_apis: []
+
+# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.get_diagnostics_for_symbol") to include in the REPL
+# that would otherwise be disabled (particularly optional methods, which are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+included_apis: []
+
+# list of mode names that are to be activated by default, overriding the setting in the global configuration.
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
+# for this project.
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+default_modes:
+
+# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+added_modes:
+
+# initial prompt for the project. It will always be given to the LLM upon activating the project
+# (contrary to the memories, which are loaded on demand).
+initial_prompt: ""
+
+# time budget (seconds) per tool call for the retrieval of additional symbol information
+# such as docstrings or parameter information.
+# This overrides the corresponding setting in the global configuration; see the documentation there.
+# If null or missing, use the setting from the global configuration.
+symbol_info_budget:
+
+# list of regex patterns which, when matched, mark a memory entry as read‑only.
+# Extends the list from the global configuration, merging the two lists.
+read_only_memory_patterns: []
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []

docker-compose.yml

@@ -27,7 +27,8 @@ services:
       --flash-attn on
       --cache-type-k ${LLAMA_KV_CACHE_TYPE:-q8_0}
       --cache-type-v ${LLAMA_KV_CACHE_TYPE:-q8_0}
-      --cache-ram ${LLAMA_CACHE_RAM:-24576}
+      --cache-ram ${LLAMA_CACHE_RAM:-8192}
+      --no-cache-idle-slots
       ${LLAMA_SPEC_ARGS:-}
       --jinja
     # see docs/architecture.md#llama-server (flags), docs/network-access.md#backends (no host port)

docs/architecture.md

@@ -26,10 +26,10 @@ The main model's backend. It is always running. Lazytainer used to idle-stop it,
 - **Per-slot context = `LLAMA_CTX_SIZE / LLAMA_PARALLEL`.** llama.cpp divides the context across slots, and each request gets one slot's share. Every coding CLI's context-window setting must use this number, not the raw context size. Real sessions already fill a large share of it, so check that the per-slot figure stays comfortably above observed usage before lowering either value.
 - **No `--cache-reuse`:** this model's context doesn't support it. llama-server logs "cache_reuse is not supported by this context, it will be disabled" at startup, so the flag was dropped. Re-prefill avoidance comes from `--cache-ram` (below) and OmniRoute's prompt-cache affinity (a dashboard setting, on by default), which keeps a conversation on the same slot. The earlier fix for the gateway's non-ping SSE timeout ([`research/omniroute-non-ping-sse-stream-timeout.md`](research/omniroute-non-ping-sse-stream-timeout.md)) assumed `--cache-reuse` worked, so long re-prefills can still hit that timeout.
 - **Testing for token corruption** (typos, broken tool-call keys): flip one key at a time in the server's `.env`, then run `docker compose up -d llama-server`. Blank `LLAMA_SPEC_ARGS` rules speculative decoding in or out, `LLAMA_KV_CACHE_TYPE=f16` does the same for the quantized KV cache (lower `LLAMA_CTX_SIZE` so it fits), and `LLAMA_CACHE_RAM=0` turns off the RAM prompt cache. A drafter's quant can't cause wrong tokens: the target model verifies every draft token, so a worse drafter only lowers acceptance.
-- **`LLAMA_CACHE_RAM`** (`--cache-ram`, MiB) is the host-RAM budget for saved prompt-cache entries. When a slot is reused, llama.cpp parks the old prompt's state in RAM so it can be restored instead of re-prefilled. The llama.cpp default is too small for ~40k-token agent prompts (entries are ~2 GiB each), so entries got evicted and requests re-prefilled in full. Raising it is the only place spare DDR RAM helps; it does not speed up decode. Keep it well below the host's free RAM. Evidence and the A/B method: [`research/qwen3.8-27b-throughput-tuning.md`](research/qwen3.8-27b-throughput-tuning.md).
+- **`LLAMA_CACHE_RAM`** (`--cache-ram`, MiB) is the host-RAM budget for saved prompt-cache entries. When a slot is reused, llama.cpp parks the old prompt's state in RAM so it can be restored instead of re-prefilled. The llama.cpp default is too small for ~40k-token agent prompts (entries are ~2 GiB each), so entries got evicted and requests re-prefilled in full. Raising it is the only place spare DDR RAM helps; it does not speed up decode. Keep it well below the host's free RAM. `--no-cache-idle-slots` stays on while [llama.cpp#27148](https://github.com/ggml-org/llama.cpp/issues/27148) is open: with the RAM cache on, an LRU-picked slot can keep an unrelated conversation's state, which showed up as misspelled output. With `--kv-unified`, turning off idle-slot caching closes that path and keeps the RAM cache ([diagnosis](research/inference-engine-spec-decoding-multi-gpu.md)). Evidence and the A/B method: [`research/qwen3.8-27b-throughput-tuning.md`](research/qwen3.8-27b-throughput-tuning.md).
 - **`--jinja`** enables the chat template that tool calling depends on. Without it, tool-use requests fail outright.
 - **Tool-calling reliability** of this model family on llama.cpp has open upstream parser bugs: [`research/qwen3.8-27b-tool-calling.md`](research/qwen3.8-27b-tool-calling.md).
-- **Speculative decoding (DFlash2):** all its flags live in one key, `LLAMA_SPEC_ARGS`: `--spec-type draft-dflash`, the drafter GGUF (`-md`, fetched by the downloader from [`models.yml`](../models.yml)) and the draft length (`--spec-draft-n-max`). One key means a single `.env` edit swaps the drafter or turns speculation off (leave it blank) for an A/B test. A blank server value differs from `.env.example`, so `update.sh` will ask about it on every run until it's restored ([operations → Env sync](operations.md#env-sync)). It replaced MTP (draft heads inside the target GGUF). Upstream: [llama.cpp#27342](https://github.com/ggml-org/llama.cpp/pull/27342). Draft acceptance can collapse at high slot counts ([llama.cpp#27117](https://github.com/ggml-org/llama.cpp/issues/27117)), another reason `LLAMA_PARALLEL` stays low.
+- **Speculative decoding (DFlash2):** all its flags live in one key, `LLAMA_SPEC_ARGS`: `--spec-type draft-dflash`, the drafter GGUF (`-md`, fetched by the downloader from [`models.yml`](../models.yml)) the draft length (`--spec-draft-n-max`), and `--spec-draft-p-min`, which ends a draft early once DFlash2's own confidence in the next token drops below the threshold. Long thinking output at the recommended sampling temperature gets low acceptance, so short confident drafts waste less verify work than long ones ([`common/speculative.cpp`](https://github.com/ggml-org/llama.cpp/blob/master/common/speculative.cpp), DFlash2 lattice walk). One key means a single `.env` edit swaps the drafter or turns speculation off (leave it blank) for an A/B test. A blank server value differs from `.env.example`, so `update.sh` will ask about it on every run until it's restored ([operations → Env sync](operations.md#env-sync)). It replaced MTP (draft heads inside the target GGUF). Upstream: [llama.cpp#27342](https://github.com/ggml-org/llama.cpp/pull/27342). Draft acceptance can collapse at high slot counts ([llama.cpp#27117](https://github.com/ggml-org/llama.cpp/issues/27117)), another reason `LLAMA_PARALLEL` stays low.
 - **No classifier backend:** the separate `qwen-classifier` was removed to free VRAM for the drafter. Anything that needs a small "fast" model must use the main model.
 - **No host port**: see [network access → Backends](network-access.md#backends).
 

docs/operations.md

@@ -1,6 +1,6 @@
 # Operations
 
-> **Source of truth:** [`scripts/update.sh`](../scripts/update.sh), [`.env.example`](../.env.example), [`models.yml`](../models.yml), and the `speed-bench` service in [`docker-compose.yml`](../docker-compose.yml) with its [`speed-bench/Dockerfile`](../speed-bench/Dockerfile) and [`scripts/bench.sh`](../scripts/bench.sh). This page explains *what the script does and why*; the steps and defaults live in the source. If they disagree, the source wins.
+> **Source of truth:** [`scripts/update.sh`](../scripts/update.sh), [`scripts/redownload-model.sh`](../scripts/redownload-model.sh), [`.env.example`](../.env.example), [`models.yml`](../models.yml), and the `speed-bench` service in [`docker-compose.yml`](../docker-compose.yml) with its [`speed-bench/Dockerfile`](../speed-bench/Dockerfile) and [`scripts/bench.sh`](../scripts/bench.sh). This page explains *what the script does and why*; the steps and defaults live in the source. If they disagree, the source wins.
 
 The stack runs on the **server**. After any change to the repo, run `./scripts/update.sh` there: it is the one command that brings the running stack in sync. It's safe to re-run at any time, and it only recreates what changed.
 
@@ -45,6 +45,13 @@ One `downloader` service in the `tools` compose profile fetches every file liste
 - **Files live in the named `models` volume**, not a host bind-mount. The downloader runs as root because the named volume is created root-owned.
 - **It needs internet on each run**: it installs `curl` and `yq` from Alpine's package mirror before reading the list.
 
+### Forcing a model re-download
+
+The skip check means `update.sh` never picks up an upstream change to a file that's already on disk (for example, a re-uploaded GGUF with the same name). [`scripts/redownload-model.sh`](../scripts/redownload-model.sh) forces it: with no argument it refreshes `LLAMA_MODEL_FILE` from `.env`, or pass a `file` name from `models.yml` (for example the drafter). It reuses the compose downloader, so the URL still lives only in `models.yml`.
+
+- The old file is renamed to `<file>.old` and deleted only after the new download succeeds. If the download fails or is interrupted, the old file is put back. This needs room for both copies in the `models` volume at once.
+- It ends by restarting `llama-server`, because the running process keeps the old file mapped until it restarts.
+
 ## Speed bench
 
 `speed-bench` runs llama.cpp's [SPEED-Bench client](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/bench/speed-bench/README.md) against the running `llama-server`. It measures decode speed (`avg_pred_t/s`), latency and draft acceptance per category. Use it to compare drafters and their flags with numbers instead of gut feel. It's in the `tools` profile, so `update.sh` never builds or runs it. The first run builds the image.

docs/research/README.md

@@ -29,6 +29,7 @@ Each file here is a **dated snapshot**: research or a decision record as it stoo
 | [proxy-shadow-pricing.md](proxy-shadow-pricing.md) | Superseded by the OmniRoute migration | Shadow cost was wired into LiteLLM config, and nothing implements it now |
 | [proxy-tool-choice.md](proxy-tool-choice.md) | Superseded by the OmniRoute migration | Chose LiteLLM. It mentions Open WebUI, which has been removed |
 | [inference-engine-spec-decoding-multi-gpu.md](inference-engine-spec-decoding-multi-gpu.md) | Current | llama.cpp vs. vLLM vs. SGLang on R9700: drafters, multi-GPU, licenses, `--cache-ram` misspelling bug. Not benchmarked |
+| [qwen3.8-27b-decode-speed.md](qwen3.8-27b-decode-speed.md) | Current | Decode tokens/s: why DFlash2 acceptance and speed fall with depth (draft length is 5, not 7), n_max/p_min, KV and slot costs, bandwidth ceiling, ordered experiment plan. Builds on the throughput-tuning note. Proposals only, nothing applied |
 | [qwen3.8-27b-quant.md](qwen3.8-27b-quant.md) | Current | Main model quant and VRAM budget |
 | [qwen3.8-27b-throughput-tuning.md](qwen3.8-27b-throughput-tuning.md) | Current | Tokens/s tuning from the server log, DDR RAM as a cache tier, comparison with five outside R9700 configs. Proposals only, nothing applied |
 | [qwen3.8-27b-tool-calling.md](qwen3.8-27b-tool-calling.md) | Current | Tool-calling parser risk for the main model |

docs/research/qwen3.8-27b-decode-speed.md

@@ -0,0 +1,226 @@
+# Research: decode tokens/s on the R9700 (llama-server, Qwen3.8-27B + DFlash2)
+
+**Date:** 2026-10-20
+
+**Question:** Why is decode at ~52k context about plain-decode speed (26-28 t/s) when an older, shorter-context log averaged 45.9 t/s, and what can be changed to raise decode (and prefill where relevant)? Evidence: `llmserver.log` (re-analysed per request) plus llama.cpp source.
+
+**Scope note:** research only. Nothing in compose or `.env.example` was changed and nothing was deployed. This note builds on [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md) (cache/prefill ranking, DDR tier, five outside configs) and [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) (quant choice, VRAM budget). It does not repeat them, and it **corrects one premise of the first** (see finding 1).
+
+**Labels.** **[V]** verified from source (URL pinned to a commit, or a repo file:line). **[L]** computed from `llmserver.log` by me. **[R]** reported by a third party (a page or thread, not re-run). **[I]** inferred, not measured. WebFetch summaries of web pages were produced by a small model, so every number taken from a page is **[R]** even when the page is first-party.
+
+**Source pin.** llama.cpp `master` at `839a572a4cdecdb80d211297b8e7d7323a6e99d2` (fetched 2026-10-20). Links below point at that commit. The running image (`server-rocm`) may be older than this commit. Its build string is not in the logs I have, so every "[V]" is "true on that commit".
+
+## TL;DR
+
+| # | Finding | Label |
+|---|---|---|
+| 1 | The draft length has been **5, not 7**, in the old log too. All 306 `draft acceptance` summaries have `generated` divisible by 5 and `mean len = 1 + accepted / (generated / 5)`. So `n_max=5` in the new startup log is the same setting, not a new clamp. The earlier note's "n_max is 7" came from compose and was wrong | [L] + [V] |
+| 2 | master clamps `n_max` to `block_size - 1` (= 7 for block_size 8). 5 is below that, so no clamp fired. 5 must have been **requested**: most likely `LLAMA_SPEC_N_MAX=5` in the server's `.env` (the repo `.env.example` default differs, and `.env` is not in the repo) | [V] + [I] |
+| 3 | Decode speed tracks acceptance almost linearly (r = 0.97 over 256 requests) and acceptance falls with depth: 0.75 at <8k, 0.57 at 8-20k, 0.42 at 20-40k, 0.40 at 40-60k, while decode falls 64.6 to 31.9 t/s. The 45.9 t/s average was dominated by short-context requests | [L] |
+| 4 | Bandwidth ceiling for plain decode: 17.6 GB / 640 GB/s = 27.5 ms = **36.4 t/s** (33.0 t/s at 52k depth with q8_0 KV). At the 79-81 % efficiency best seen for dense models on this card that is 26-27 t/s, which is exactly the measured "plain" rate | [I] on [R] inputs |
+| 5 | A spec step costs 74-94 ms in the old log, against a ~28-30 ms weight-streaming floor. About 45 ms per step is fixed overhead (drafter, 6-wide verify, sync, sampling), so speculation only pays when acceptance is high | [L] + [I] |
+| 6 | DFlash2 draws its draft greedily and gives the server no draft distribution, so at temp > 0 a draft token is accepted only if the target's own sampled token equals it. Sampler settings on the client therefore change acceptance directly | [V] |
+| 7 | `GPU_MAX_HW_QUEUES=1` is free for default llama.cpp: multi-stream graph execution is opt-in (`GGML_CUDA_GRAPH_OPT=1`) | [V] |
+
+Top 3 experiments (details in "Experiment plan"): **A** sweep `--spec-draft-n-max` 3, 4, 5, 7 and add `--spec-draft-p-min`; **B** measure acceptance versus client sampler settings with a fixed prompt (temp, top-p/top-k, penalties); **C** decompose the 74-94 ms step with `llama-bench` (pp6 at depth vs tg1) and `rocm-smi` clocks, before touching KV type, slots or quant.
+
+## 1. DFlash2: why acceptance and speedup vary
+
+### 1.1 The n_max 5 vs 7 discrepancy
+
+- The init line prints `n_max=%d` from `this->params.n_max` **before** the clamp: [`common/speculative.cpp#L1086`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1086). The clamp is [`#L1090-L1099`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1090): `n_draft_max = (is_dspark && sample_from_anchor) ? block_size : block_size - 1`, with the warning `requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d`. **[V]**
+- For this drafter (`block_size=8`, DFlash2 is not DSpark) the limit is 7. So a requested 7 passes unclamped and a requested 5 prints 5. The init line therefore shows the **requested** value; no clamp explains 5. **[V]** (on that commit)
+- `n_extract=5` in the log is `target_layer_ids_n`, the number of target layers whose features feed the drafter ([`#L1040-L1044`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1040)). It is not used in the clamp, so the 5 = 5 match is a coincidence on this commit. **[V]** An older build could clamp differently; I could not check history for this. **[I]**
+- Log proof that 5 is the real draft length. The server prints `mean len = 1 + accepted / verif_steps` ([`tools/server/server-context.cpp#L704-L725`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L704)). With `verif_steps = generated / 5` this reproduces the printed mean len in all 306 summaries; with 7 it matches none. Examples: `llmserver.log:3` (47 accepted / 105 generated, 21 steps, 47/21 + 1 = 3.24), `:36` (64/170, 34 steps, 2.88), `:114` (828/2050, 410 steps, 3.02). **[L]**
+- To confirm on the server: `grep -n SPEC_N_MAX .env` and `docker logs llama-server | grep -E "n_max|exceeds the trained"`. **[I]**, not run.
+
+### 1.2 What the drafter does per step
+
+- The drafter input is `[id_last, <mask> * n_max]` and one forward pass denoises the whole block. DFlash2 then greedily walks a candidate lattice (`selector_top_k` candidates per position, argmax per step): [`#L1296-L1310`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1296) and [`#L1336-L1366`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1336). The block size in the drafter batch is `n_max + 1`, so a smaller `n_max` shrinks the drafter batch too. **[V]**
+- `--spec-draft-p-min` on DFlash2: stop extending the draft at the first position where softmax(selector scores) at the argmax is below `p_min` ([`#L1349-L1358`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1349)). It is a per-position stop, default 0.00 ([`common/common.h#L333`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L333), [`common/arg.cpp#L4248`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4248)). The git log shows DFlash2-specific p_min arrived with the DFlash2 PR ([#27342](https://github.com/ggml-org/llama.cpp/pull/27342), commit titles "Add p_min in DFlash2" per the PR page **[R]**). **[V]** for the code. The `generated` multiples of 5 in the log confirm p_min was 0 in the old run. **[L]**
+- `n_min` (`--spec-draft-n-min`, default 0): a draft shorter than `n_min` is discarded ([`#L1362-L1364`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1362)). **[V]**
+- `sample_from_anchor=true` only changes the layout for DSpark drafters; for DFlash2 the block is `n_max + 1` ([`#L1298`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1298)). **[V]**
+- `--spec-draft-sampling probabilistic` ([`arg.cpp#L4264`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4264)) is documented as sampling the draft and verifying by rejection. The DFlash2 branch never fills the draft distribution (`result_q`), and the server then falls back to plain sample-and-match: `use_rejection = use_spec_rejection() && !spec_draft_q.empty()` ([`server-context.cpp#L4537`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L4537)). So that flag does nothing for DFlash2. **[V]** (read of the code path; not run).
+
+### 1.3 Why acceptance varies
+
+1. **Sampling.** With the DFlash2 drafter, a draft token is accepted iff the target's sampled token equals it ([`common/sampling.cpp#L690-L720`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/sampling.cpp#L690), `draft[i] != id` breaks). At temp 0 the target token is its argmax, the best case for matching a greedy draft. At temp > 0 it falls with flatness of the target distribution and with anything that reshapes it (top-p/top-k, repetition or presence penalties). **[V]** for the mechanism. The size of the effect for this workload is unmeasured. **[I]**
+   - Contradiction to resolve: the earlier note cites a third-party claim that greedy sampling "collapses" DFlash2 acceptance. The source reading says the opposite for the server path. Treat that claim as unverified until experiment B settles it.
+   - Server-side the temp > 0 test is `task->params.sampling.temp > 0` ([`server-context.cpp#L508`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L508)). The drafter itself ignores temperature. **[V]**
+2. **Context depth.** In the old log, acceptance halves from <8k to >40k tokens (0.75 to 0.40, table below). A PR commenter also reported code prompts decaying faster with context length than prose ([PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) comments) **[R]**. The model card gives no context length for the drafter. **[R]**
+3. **Content type.** Not separable in this log (no prompt/response classes). The authors' acceptance lengths (mean tokens per verify step, 7 drafted, block size 8, temp 1.0 / top-p 0.95 / top-k 20, SGLang on an H200) are GSM8K 5.46, MATH-500 5.28, HumanEval 4.39, MBPP 4.79 ([z-lab/Qwen3.8-27B-DFlash2](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2), via a search summary) **[R]**. Your mean len of 2.9-4.9 (max possible 6 at n_max 5) is below that, partly because those are short prompts. **[I]**
+4. **Concurrency.** Issue [#27117](https://github.com/ggml-org/llama.cpp/issues/27117) (gfx1151, closed "not planned"): acceptance collapses as the number of concurrently drafting sequences grows (0.51-0.66 at 4 active, 0.06-0.49 at 16), and `--spec-draft-n-max 1` recovered it. **[R]** Your two slots are far below that regime, but the bimodal per-slot acceptance in that issue is worth watching if both slots decode at once. **[I]**
+
+### 1.4 Authors' recommended values
+
+- Model card quick start: `--spec-type draft-dflash --spec-draft-n-max 7` ([z-lab/Qwen3.8-27B-DFlash2-GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF)) **[R]**. No block size, temperature or p_min advice is given there. The card says the output is lossless (greedy identical, sampling distribution preserved).
+- A PR commenter on an RTX 3090 used `--spec-draft-n-max 4` and reported that 7 was slower: DFlash2 at n=4 averaged 62.21 t/s at 0.6153 acceptance, MTP 57.79 t/s at 0.6317 ([PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) comments) **[R]**.
+- Drafter quant does not matter for acceptance: acceptance length 5.28 (BF16), 5.13 (Q8_0), 5.39 (Q4_K_M) on 8 GSM8K prompts; the card itself implies Q4_K_M above BF16 is noise. Q4_K_M is ~1.1 GB, so it also costs the least bandwidth. **[R]**
+
+### 1.5 What the old log shows (re-analysis)
+
+Per-request join of `stop processing: n_tokens` (final depth), `eval time` t/s and `draft acceptance`, requests with >= 200 generated tokens, 256 requests (script: ad-hoc Python over `llmserver.log`, not committed). Draft length 5, so tokens per verify step is 1 + 5 x acceptance.
+
+| Final depth | n | Decode t/s (mean) | Acceptance (mean) | Tokens/step | Step time (tokens/step / t/s) |
+|---|---|---|---|---|---|
+| < 8k | 125 | 64.6 | 0.75 | 4.75 | ~74 ms |
+| 8-20k | 28 | 44.2 | 0.57 | 3.85 | ~87 ms |
+| 20-40k | 17 | 34.6 | 0.42 | 3.10 | ~90 ms |
+| 40-60k | 86 | 31.9 | 0.40 | 3.00 | ~94 ms |
+
+- Correlation of per-request decode t/s with acceptance: r = 0.97, slope ~90 t/s per 1.0 acceptance. **[L]**
+- Step time rises only ~20 ms from 0 to 50k depth, while tokens per step drops 4.75 to 3.0. So about **80 % of the decode loss with depth is acceptance, not attention cost**. **[L]** (derived, mean-of-ratios caveat).
+- The new run (26-28 t/s at ~52k, tg_3s 20-33) is at or just below the 40-60k bucket (31.9), consistent with the same acceptance regime. **[L]**
+
+## 2. Decode speed versus context depth
+
+### 2.1 What the attention costs here
+
+- Only 16 of 64 layers keep a KV cache ([`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) "Model identity"). KV bytes per token: 16 layers x 2 x 4 heads x 256 x element size = 65,536 B (f16), 34,816 B (q8_0, 1.0625 B/element), 18,432 B (q4_0, 0.5625 B/element). At 52k tokens: 3.4 GB (f16), 1.81 GB (q8_0), 0.96 GB (q4_0). Against 17.6 GB of weights read per token, q8_0 KV adds ~10 % bytes at 52k, f16 ~19 %, q4_0 ~5 %. **[I]** (arithmetic from the quant note's architecture facts.)
+- So KV precision can move plain decode at 52k by at most a few percent. Going to q4_0 would buy ~1 GB less read (~5 %) at an unmeasured quality cost. Do not expect it to fix the 2x gap. **[I]**
+
+### 2.2 Which flash-attention kernel runs on gfx1201
+
+From [`ggml/src/ggml-cuda/fattn.cu#L540-L722`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L540) (`ggml_cuda_get_best_fattn_kernel`), applied to head_dim 256, 24 query heads, 4 KV heads (gqa_ratio 6, effective ratio 2), K/V q8_0:
+
+| Query batch (tokens) | Kernel on gfx1201 | Why (source) |
+|---|---|---|
+| 1 (plain decode) | VEC | WMMA needs `Q->ne[1] * gqa_ratio_eff > 16` for head_dim > 128 ([`#L698-L702`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L698)), so 1 x 2 fails. For quantized K/V the vector kernel is chosen when `Q->ne[1] <= 2` ([`#L714-L720`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L714)). Reads q8_0 directly. |
+| 2 | VEC (quantized) | same rule |
+| 3 to 8 (verify batch of a spec step) | **TILE** | 8 x 2 = 16 is not > 16, so no WMMA; quantized KV is not eligible for VEC above 2 tokens. |
+
+- TILE and MMA kernels "need f16 K/V" ([`#L735-L740`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L735)): with a quantized cache the K and V views are converted to f16 scratch before the kernel runs. So a verify step at 52k converts ~100M q8_0 elements per layer (K plus V) to f16 on 16 layers. Rough extra traffic: read 1.8 GB plus write 3.4 GB plus read 3.4 GB per step, ~8.6 GB, ~13 ms at 640 GB/s. **[I]** (the conversion and kernel choice are [V]; the traffic estimate and its cost are inferred, not profiled). That is consistent with the ~20 ms step-time rise from 0 to 50k in 1.5. With f16 KV the conversion disappears (TILE reads f16 directly) but the cache is 1.9x larger, so the net is not obvious. **[I]**
+- No kernel-level bug report for gfx1201 + quantized KV turned up in the sources I read. The only RDNA4 data point is [R]: on RADV (Vulkan) q8_0 vs q4_0 KV was within +-0.5 % ([discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043)). That is a different backend and says nothing about the HIP path.
+
+### 2.3 `--parallel 2` and `--kv-unified`
+
+- With `--kv-unified` there is one KV stream. `get_n_kv` pads `cells.used_max_p1()`, the highest occupied cell across **all** sequences ([`src/llama-kv-cache.cpp#L1282-L1292`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-kv-cache.cpp#L1282)). The flash-attention kernels only skip fully masked tiles at the **end** of the range (`flash_attn_mask_to_KV_max`, [`fattn-common.cuh#L671-L723`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn-common.cuh#L671)). So when the other slot holds a long prompt in higher cells, slot 0's decode may read and mask-skip past tens of thousands of foreign cells. **[V]** for both pieces, **[I]** for the combined cost (not profiled; which cells each slot occupies is not logged).
+- This matches the pattern in the older log (requests alternate slot 0 / slot 1, each ~40-50k tokens, [throughput note](qwen3.8-27b-throughput-tuning.md) "Pattern at `:380`-`:470`"), but no measurement separates it. **Experiment D** below tests it.
+- `--kv-unified-per-slot N` exists ([`arg.cpp#L1652`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L1652)) and `--no-kv-unified` gives one stream per slot ([`arg.cpp#L1727-L1734`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L1727)). Non-unified mode splits `--ctx-size` evenly, so one slot could no longer use more than half the context. **[V]**
+
+### 2.4 Checkpoints and recurrent state during decode
+
+- The target is `qwen35` architecture, which `llm_arch_supports_rs_rollback` lists ([`src/llama-arch.cpp#L1256-L1262`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-arch.cpp#L1256)). I am assuming Qwen3.8 loads as that arch; the name was not verified. **[I]**
+- For DFlash/DSpark the server requests `n_rs_seq = draft.n_max` recurrent-state snapshots per sequence ([`common/common.h#L401-L407`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L401); [`src/llama-context.cpp#L191-L195`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-context.cpp#L191)). The server creates a **speculative checkpoint** only if `draft.size() > llama_n_rs_seq(ctx_tgt)` ([`server-context.cpp#L3541-L3558`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L3541)), and restores one only if `n_rollback > n_rs_seq` ([`#L4559`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L4559)). With `n_rs_seq == n_max` that never happens, so a rejection is a snapshot-index rollback and not a state copy. **[V]**
+- If the build is older or the arch is not on the rollback list, the `COMMON_CONTEXT_SEQ_RM_TYPE_FULL` path copies a checkpoint every step (`use_ckpt_tgt`, [`#L3463`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L3463)); the startup line `speculative decoding will use checkpoints` ([`#L1388-L1390`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L1388)) would show it. Check the startup log for that line. **[V]** for the code, **[I]** for which case applies.
+- `--ctx-checkpoints` / `--checkpoint-min-step` (defaults 32 and 8192, [`common.h#L639-L642`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L639)) are for prompt/prefix reuse. Nothing I read ties them to the decode loop, so they should not cost decode time. **[I]**
+- The "cache_reuse is not supported by this context" startup line is the server's `llama_memory_can_shift` check ([`server-context.cpp#L1328-L1336`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L1328)). It confirms the open question in the throughput note. **[V]**
+
+## 3. Backend, HIP flags, clocks
+
+### 3.1 ROCm versus Vulkan (RADV)
+
+Everything here is **[R]** unless noted; the thread posts are self-reported and mixed in setup.
+
+| Source | Setup | Numbers |
+|---|---|---|
+| [discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043) (JohnTDI-cpu, build 8554) | R9700, RADV, Qwen3.5-27B Q4_K_M | tg128 29.07 t/s (RADV), 29.01 (AMDVLK); pp512 798 (RADV). Computed there as ~456 GB/s, 71 % of 640 GB/s; "best dense ~79-81 %" |
+| same | R9700, ROCm, 27B Q8_0, q4_0 KV, 64k ctx | pp 960 (3.7k) to 708 (58.7k) t/s; tg 42.8 to 28.0 t/s over the same range (MTP use not stated, and 42.8 t/s is above a 29 GiB Q8_0 plain-decode ceiling of ~22 t/s, so it is almost certainly with speculation) |
+| same | Windows, HIP, gfx1201 | pp512 1106-1211; tg128 29.07; "Vulkan decode still wins" |
+| same | R9700, Qwen3.6-27B UD-Q4_K_XL, q4_0 KV, MTP (build b9203) | tg128 28.2 baseline, 47.4 with `mtp-3` |
+| [discussion #20881](https://github.com/ggml-org/llama.cpp/discussions/20881) | R9700 and others | MoE Qwen3-Coder-30B-A3B Q4_0 ~196 t/s on RADV; one ROCm segfault report on Qwen3.5-35B-A3B |
+| [throughput note](qwen3.8-27b-throughput-tuning.md) (cat5edopeHA) | R9700, MTP n3 | Vulkan 68.5 / ROCm 55.1 short, 50.5 / 40.2 at 32k, 40.9 / 31.2 at 64k |
+
+- A search summary also cites a blog (not fetched) with ROCm ahead for dense 27B decode (42.8 vs 29.1), the opposite of cat5edopeHA. The sources conflict, so only a local A/B decides. **[R]**
+- DFlash2 on Vulkan: not verified. The DFlash graph and feature extraction live in common code and the model graph, so it may work, but I did not check that a Vulkan build runs `draft-dflash`. **[I]**
+- Prefill: HIP uses MMQ on RDNA4 unconditionally (`ggml_cuda_should_use_mmq` returns true for RDNA4, [`mmq.cu#L468-L471`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmq.cu#L468)), so `GGML_CUDA_FORCE_MMQ` changes nothing on gfx1201. **[V]**
+
+### 3.2 Decode kernels for the verify batch
+
+- Weights are Q4_K-family. Matrix multiplies with up to 8 columns go to MMVQ (`MMVQ_MAX_BATCH_SIZE 8`, [`mmvq.cuh`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmvq.cuh)). RDNA4 has no per-type override, so `ne11 <= 8` uses MMVQ ([`mmvq.cu#L318-L430`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmvq.cu#L318)). The source comment says k-quants "cost more to decode and mvq redoes that per column". **[V]** So the verify batch of 6 (n_max 5) or 8 (n_max 7) pays dequant ALU work per column; cost grows with batch width even though weight bytes are read once. **[I]** (the per-column cost on gfx1201 was not measured). This is the mechanism behind "lower n_max can win": it shrinks both the MMVQ columns and the drafter batch.
+- Batch 9 or more would fall to MMQ, which is not reachable: `n_max` is capped at 7 so the batch is at most 8. **[V]**
+
+### 3.3 HIP flags and environment variables
+
+| Item | Verdict | Source |
+|---|---|---|
+| `GGML_HIP_GRAPHS` | Already ON in the CMake default and the official ROCm Dockerfile does not turn it off. Variable graph shapes force a re-warmup: any change in node properties resets `warmup_complete` ([`ggml-cuda.cu#L4806-L4845`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/ggml-cuda.cu#L4806)). With `p_min > 0` the verify batch width varies, so graphs may be re-captured more often. Debug with `GGML_CUDA_GRAPH_DEBUG=1`; disable with `GGML_CUDA_DISABLE_GRAPHS=1`. | [V] ([`ggml/CMakeLists.txt#L217`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/CMakeLists.txt#L217), [`rocm.Dockerfile`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/.devops/rocm.Dockerfile)); effect on speed **[I]** |
+| `GGML_CUDA_FORCE_MMQ` | No effect on RDNA4 (see 3.1) | [V] |
+| `GGML_CUDA_GRAPH_OPT=1` | Opt-in multi-stream concurrent branches ([`ggml-cuda.cu#L5016-L5030`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/ggml-cuda.cu#L5016)). Off by default | [V] |
+| `GPU_MAX_HW_QUEUES=1` | HIP caps hardware queues per process per device (default 4, [HIP env docs](https://rocm.docs.amd.com/projects/HIP/en/docs-7.2.3/reference/env_variables.html)). llama.cpp issues all kernels on one stream unless `GGML_CUDA_GRAPH_OPT=1`, so one queue should cost nothing for decode. The repo set it for [ROCm/ROCm#5706](https://github.com/ROCm/ROCm/issues/5706) (`docs/architecture.md#gpu-access`). Do not combine `GGML_CUDA_GRAPH_OPT=1` with it without a test | [R] for the doc, [V] for the llama.cpp side, **[I]** for "no decode cost" |
+| `-bs, --backend-sampling` | Experimental, default off ([`arg.cpp#L2310`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L2310)). Moves target sampling onto the GPU, which would remove the CPU sampling of 6 logit rows per step. Interplay with speculation is unverified | [V] flag exists; benefit **[I]** |
+| `rocm-smi` perf level / compute profile | In #21043: "ROCm `rocm-smi` high vs auto: auto was faster" on a single card; a LACT 330 W cap with COMPUTE profile gave tg 102 to 110 on a dual-card MoE run; `power_dpm_force_performance_level=high` gave +14 % tg on one Vulkan run. Mixed, hardware and backend differ | [R] |
+| PCIe ASPM `pcie_aspm.policy=performance` | Same thread: dense decode on RADV 29.30 to 32.46 (+10.8 %), one user, not reproduced on eGPU or dual-slot. It is a runtime sysfs switch, so it is cheap to try | [R] |
+| MCLK not reaching max under load | Same thread, kernel 6.17, RADV | [R]; check with `rocm-smi` during decode, **[I]** |
+
+### 3.4 Threads, mmap, mlock, poll
+
+- With every layer on the GPU the CPU is only a scheduler. #21043 tested `-t`, `-b`, `-mmp 0`, `--prio 2`, CPU pinning, hugepages and `nice` on RADV and found all within +-0.3 % **[R]**. The server default `--poll` level is 50 ([`common.h#L77`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L77)) and `-t` was -1 (16 here). I found no evidence these matter for a GPU-resident run, so do not spend experiments on them. **[I]**
+- One exception worth knowing: speculative drafting has its own thread/poll flags (`--spec-draft-threads`, `--spec-draft-poll`, [`arg.cpp`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp)), but the drafter is on the GPU too, so they should be equally inert. **[I]**
+
+## 4. Quant and model options
+
+Inputs: GGUF sizes from [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) (UD-Q4_K_XL 17.6 GB, UD-Q4_K_M 16.5, UD-Q4_K_S 15.4) and the throughput note (UD-IQ4_XS 13.3 GB). Bandwidth 640 GB/s is the card spec as quoted in #21043 **[R]**. Weights only; 52k q8_0 KV adds 1.81 GB (2.1).
+
+| Quant | Bytes read/token | Ceiling at 640 GB/s | At 80 % efficiency | Same with 52k q8_0 KV (ceiling / 80 %) |
+|---|---|---|---|---|
+| UD-Q4_K_XL (running) | 17.6 GB | 36.4 t/s (27.5 ms) | 29.1 | 33.0 / 26.4 |
+| UD-Q4_K_M | 16.5 GB | 38.8 | 31.0 | 35.0 / 28.0 |
+| UD-Q4_K_S | 15.4 GB | 41.6 | 33.2 | 37.2 / 29.8 |
+| UD-IQ4_XS | 13.3 GB | 48.1 | 38.5 | 43.5 / 34.8 |
+
+- All **[I]** (arithmetic). The 80 % factor is the "best dense ~79-81 %" figure in #21043 **[R]**. The measured plain-decode rate at depth, 26-28 t/s, sits on the 52k column for Q4_K_XL (26.4), so the card is already near its bandwidth limit for plain decode. There is no hidden 2x in the non-speculative path.
+- A smaller quant gains at most ~13 % (Q4_K_M) to ~32 % (IQ4_XS) on the plain-decode ceiling and also shrinks the spec verify weight read. Quality cost is not measured here. The IQ4 family dequantizes slower than Q4_K, so the real gain is below the ceiling ratio. **[I]** Test with KL divergence or perplexity on a small fixed corpus (`llama-perplexity`) before any switch.
+- Drafter quant: Q4_K_M at ~1.1 GB is already the smallest tested, with acceptance as good as BF16 (1.4). Nothing to gain. **[R]**
+- Per spec step the drafter adds ~1.1 GB of reads (~1.7 ms at the ceiling), small against the ~45 ms fixed overhead found in 1.5. **[I]**
+
+## 5. Where the step time goes (open)
+
+Old-log step times are 74 ms at <8k and 94 ms at 40-60k ([1.5](#15-what-the-old-log-shows-re-analysis)). The weight-streaming floor at 80 % efficiency is ~34 ms. The remaining ~40-60 ms per step is not explained by any single item I could verify. Candidate contributors, none measured:
+
+1. 6-column MMVQ with k-quant dequant per column (3.2).
+2. Drafter forward over `n_max + 1` positions plus injecting 5 layers of target features per accepted token (`n_extract=5`, [`speculative.cpp#L1040`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1040)), including the encoder fused into KV injection ([commit 662a0b0](https://github.com/ggml-org/llama.cpp/commit/662a0b012)).
+3. CPU sampling of 6 full-vocab logit rows plus device-to-host copies per step (`-bs` is off).
+4. Host synchronisation between drafter and target per step (two contexts, no overlap).
+5. Quantized-to-f16 KV conversion for the TILE kernel at depth (2.2).
+
+Experiment C isolates 1 and 5 without the server; the rest need a profile (`rocprofv3`), which is out of scope here.
+
+## 6. Experiment plan
+
+Rules: one change per run, cold slot, same prompt, restart the server between configs with `./scripts/update.sh` on the server (this box cannot deploy). All numbers come from the server log lines `prompt eval time`, `eval time` and `draft acceptance` (`llmserver.log:32-36` show the format).
+
+### Benchmark protocol (minimal, comparable)
+
+1. **Fixed prompts** (kept in the repo's scratch area, not committed): P-code = ~8k tokens of source plus the instruction "continue the file" and P-code-52k = the same material padded to ~52k tokens; P-reason = a reasoning question over the same 52k context. Same bytes every run.
+2. **Fixed request:** `n_predict=512` (400+ tokens so the 3-s `tg` window averages out), `seed` fixed, `cache_prompt` false for the first request so prefill is cold, then the same request again with the prompt cache warm only if you want decode in isolation.
+3. **Sampler:** run each prompt twice, once with the production client settings (record them from the request body) and once with `temperature=0`. Record the sampler fields in the run name.
+4. **Repeats:** N = 5 per cell, both slots idle except the request under test; discard the first.
+5. **Read** from the log for each request: prompt tokens/s (`prompt eval time`), decode t/s (`eval time`), `draft acceptance` and `mean len`. Report median and min/max, not the mean.
+6. **Decode-only control:** `llama-bench` inside a throwaway container (stop `llama-server` first, one GPU): `llama-bench -m <model> -ngl 99 -fa 1 -ctk q8_0 -ctv q8_0 -p 0 -n 256 -d 0,16384,52000 -r 3`. `-n 256` avoids the "implicit ubatch 128 on hybrid models" artifact reported with `-n 128` in #21043 **[R]**. `llama-bench` does not use speculation or the prompt cache.
+7. **Environment capture** per run: `rocm-smi --showclocks --showpower --showtemp` once during the decode (SCLK, MCLK), plus the full startup log (build string, `n_rs_seq`, checkpoint line).
+
+### Experiments, in priority order
+
+| # | Change (exact) | Expected effect | How to measure | Risk |
+|---|---|---|---|---|
+| A | `LLAMA_SPEC_N_MAX` = 3, 4, 5, 7 (one run each); then 5 with `--spec-draft-p-min 0.3` and `0.5` | Fewer wasted drafted tokens and a narrower verify batch when acceptance is ~0.4; could be neutral or negative when acceptance is high. Direction at 52k is the point; short context favors the larger value. No estimate of size without data | Decode t/s, `mean len`, `generated` (must be a multiple of the n_max, or not if p_min is on), on P-code and P-code-52k | Low. `p_min` makes batch widths vary, which can trigger graph re-capture (3.3); if so, compare with `GGML_CUDA_DISABLE_GRAPHS=1` |
+| B | No server change: send P-code-52k / P-reason with (1) production sampler, (2) `temperature=0`, (3) production minus penalties, (4) production minus top-p/top-k | Isolates how much acceptance the sampler costs (6, 1.3). A large gap says the fix is in the client/gateway sampler defaults, not llama.cpp | `draft acceptance` and `mean len` per cell; the full per-position `acc per pos` appears at trace level only | None |
+| C | Stop `llama-server`; in a container with the same image: `llama-bench ... -p 1,2,4,6,8 -n 0 -d 0,52000` and `-p 0 -n 256 -d 0,52000`; rocm-smi clocks during a server decode | Splits step time into verify-batch cost (pp6 time at depth) vs plain tg1. Shows whether verify at depth already costs ~60+ ms | `llama-bench` t/s table; compare pp6 time with measured step time | Low. Needs a short outage of the only GPU consumer |
+| D | `--parallel 1` (then `--no-kv-unified` with `--parallel 2`) on the same P-code-52k | Tells whether the second slot or the unified cache costs single-stream decode (2.3). If decode at 52k jumps, slot layout is the cost | Decode t/s on a request with the other slot idle; then with the other slot holding a 50k context | Medium. `--parallel 1` serialises agents; non-unified halves per-slot max context |
+| E | `echo performance \| sudo tee /sys/module/pcie_aspm/parameters/policy`, and check MCLK/SCLK under load; try `rocm-smi --setperflevel high` only if MCLK sags | Reported +10.8 % on one RADV dense run; unknown for ROCm. Free to try and revert | Decode t/s on P-code (plain control from C) | Low; revert with the previous policy value |
+| F | `-bs` (`--backend-sampling`) | Removes CPU sampling of the verify rows if that is a real share of the step | A/B on P-code | Medium: experimental, speculation interplay unverified |
+| G | `GGML_CUDA_DISABLE_GRAPHS=1` and, separately, `GGML_CUDA_GRAPH_DEBUG=1` to see re-warmup counts | Tells whether graph churn from the variable shapes is a cost | A/B on P-code | Low |
+| H | `-ctk q4_0 -ctv q4_0` at 52k | Saves ~0.85 GB read per token (~4-5 % on plain decode) and removes some f16 conversion traffic; quality risk | A/B on P-code-52k plus a spot-check of output quality | Medium (quality) |
+| I | `-ctk f16 -ctv f16 --ctx-size 131072` | Removes q8_0 to f16 conversion for the TILE verify kernel (2.2); cache is 1.9x larger. VRAM: f16 KV for 131072 tokens is ~8 GB | A/B on P-code-52k | Medium (halves context, VRAM) |
+| J | Smaller quant (UD-Q4_K_M or UD-IQ4_XS) from the same repo | Up to +13 % / +32 % on the plain ceiling (4); quality unmeasured | A/B plus KLD/perplexity | Medium (quality, download) |
+| K | Vulkan (RADV) image, if `draft-dflash` runs on it | Reported decode lead on some setups, but conflicting sources and prefill lag at short context | A/B with the same protocol | High (new image, unverified DFlash2 support) |
+| L | `--spec-type draft-dflash,ngram-mod` (list is accepted, [`arg.cpp#L4314-L4322`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4314)) | Extra drafts on repeated text (tool JSON, code). Interaction with the DFlash drafter is unchecked | A/B on P-code | Medium |
+
+Not worth an experiment: `-t`, `--poll`, `--no-mmap`, `--mlock`, `GGML_CUDA_FORCE_MMQ`, drafter quant (3.4, 3.1, 1.4).
+
+## Open questions
+
+- What does the server `.env` say for `LLAMA_SPEC_N_MAX`, and what does the new startup log print for `speculative decoding will use checkpoints` and `n_rs_seq`?
+- What temperature, top-p/top-k and penalties do OmniRoute clients actually send? If they differ per client, acceptance will differ per client.
+- Is the ~45 ms fixed overhead per spec step in the target verify, the drafter, or the CPU? Experiment C and a `rocprofv3` trace would answer it.
+- Does a Vulkan build run `draft-dflash` at all?
+- Which exact llama.cpp build is in the image, and is it before or after the `n_rs_seq` snapshot change?
+
+## Sources
+
+- Repo: `llmserver.log` (lines cited inline), `docker-compose.yml`, `.env.example`, `docs/architecture.md#llama-server`, `docs/research/qwen3.8-27b-throughput-tuning.md`, `docs/research/qwen3.8-27b-quant.md`.
+- llama.cpp at `839a572a4cdecdb80d211297b8e7d7323a6e99d2`: `common/speculative.cpp`, `common/arg.cpp`, `common/common.h`, `common/sampling.cpp`, `tools/server/server-context.cpp`, `src/llama-kv-cache.cpp`, `src/llama-arch.cpp`, `src/llama-context.cpp`, `src/llama-memory-recurrent.cpp`, `ggml/src/ggml-cuda/{fattn.cu,fattn-common.cuh,mmq.cu,mmvq.cu,mmvq.cuh,ggml-cuda.cu}`, `ggml/CMakeLists.txt`, `.devops/rocm.Dockerfile`. Line numbers were read from `raw.githubusercontent.com/.../master/...` on the day of fetch; if the link anchors drift, search by the quoted identifier.
+- [PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) (DFlash2), [issue #27117](https://github.com/ggml-org/llama.cpp/issues/27117), [discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043), [discussion #20881](https://github.com/ggml-org/llama.cpp/discussions/20881).
+- [z-lab/Qwen3.8-27B-DFlash2-GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF), [z-lab/Qwen3.8-27B-DFlash2](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2) (via a search summary), [HIP environment variables](https://rocm.docs.amd.com/projects/HIP/en/docs-7.2.3/reference/env_variables.html).
+- Not fetched: the Phoronix R9700 ROCm vs Vulkan review (HTTP 403), AMD ROCm RDNA4 pages beyond the HIP env page.

scripts/redownload-model.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Forces a fresh download of a model file listed in models.yml, replacing the one on disk.
+# Why and how: docs/operations.md#forcing-a-model-re-download
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [ $# -gt 1 ] || [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
+  echo "Usage: $0 [model-file]   (default: LLAMA_MODEL_FILE from .env)" >&2
+  exit 1
+fi
+
+file="${1:-}"
+if [ -z "$file" ]; then
+  [ -f .env ] || { echo ".env missing, run scripts/update.sh first" >&2; exit 1; }
+  file="$(grep -m1 '^LLAMA_MODEL_FILE=' .env | cut -d= -f2- | tr -d '\r"'"'" || true)"
+  [ -n "$file" ] || { echo "LLAMA_MODEL_FILE is not set in .env" >&2; exit 1; }
+fi
+grep -qxF "  - file: $file" models.yml || { echo "$file is not listed in models.yml" >&2; exit 1; }
+
+# Runs a snippet against the models volume; $1 in it is the model file.
+in_volume() { docker compose --profile tools run --rm --entrypoint sh downloader -c "cd /models; $1" _ "$file"; }
+
+# Put the previous file back if the download didn't finish, so a failed run never loses the working model.
+ok=false
+restore() {
+  $ok || in_volume '[ ! -f "$1.old" ] || { rm -f "$1" "$1.part"; mv "$1.old" "$1"; }' || true
+}
+trap restore EXIT
+
+echo "==> moving aside the current $file (kept until the new download succeeds)"
+in_volume '[ -f "$1" ] || { [ ! -f "$1.old" ] || mv "$1.old" "$1"; }; [ ! -f "$1" ] || mv -f "$1" "$1.old"; rm -f "$1.part"'
+
+echo "==> downloading $file"
+docker compose --profile tools run --rm downloader
+
+ok=true
+in_volume 'rm -f "$1.old"'
+
+echo "==> restarting llama-server to load the new file"
+docker compose restart llama-server