Merge pull request 'docs(research): add qwen3.8-27b decode speed analysis' (#68) from docs/decode-speed-research into main

Reviewed-on: #68

.serena/.gitignore

@@ -0,0 +1,2 @@
+/cache
+/project.local.yml

.serena/project.yml

@@ -0,0 +1,187 @@
+# the name by which the project can be referenced within Serena/when chatting with the LLM.
+project_name: "LLM-Server"
+
+# list of language servers to start when using the LSP backend; choose from:
+#   ada                    al                     angular                ansible                astro
+#   bash                   bsl                    clojure                cpp                    cpp_ccls
+#   crystal                csharp                 csharp_omnisharp       cue                    dart
+#   deno                   elixir                 elm                    erlang                 fortran
+#   fsharp                 gdscript               gleam                  go                     groovy
+#   haskell                haxe                   hlsl                   html                   java
+#   json                   julia                  julia_fatou            kotlin                 latex
+#   lean4                  lua                    lua_emmylua            luau                   markdown
+#   matlab                 msl                    nextflow               nix                    ocaml
+#   pascal                 perl                   php                    php_devsense           php_phpactor
+#   php_phpantom           powershell             python                 python_basedpyright    python_jedi
+#   python_pyrefly         python_ty              qml                    r                      rego
+#   ruby                   ruby_solargraph        rust                   scala                  scss
+#   solidity               svelte                 swift                  systemverilog          terraform
+#   toml                   typescript             typescript_tsgo        typescript_vts         vue
+#   vbnet                  wolfram                yaml                   zig
+#   (This list may be outdated; generated with scripts/print_language_list.py;
+#   For the current list, see values of the LanguageServerId enum here:
+#   https://github.com/oraios/serena/blob/main/src/solidlsp/ls_config.py)
+# For some languages, there are several alternative language servers, e.g. csharp_omnisharp, ruby_solargraph.)
+# Note:
+#   - For C, use cpp
+#   - For JavaScript, use typescript
+#   - For Angular projects, use angular (subsumes typescript+html; requires `npm install` in the project root)
+#   - For Astro projects, use astro (subsumes typescript/javascript for .astro projects; requires npm)
+#   - For Svelte projects, use svelte (subsumes typescript/javascript for .svelte projects; requires npm)
+#   - For Deno projects, use deno (serves the same .ts/.js files as typescript; requires the deno CLI on PATH)
+#   - For SCSS / Sass / plain CSS, use scss (some-sass-language-server handles all three)
+#   - For Free Pascal/Lazarus, use pascal
+#   - External Python adapters may add further registered IDs; install the adapter package first
+#     and then use its ID here, for example: example
+# Special requirements:
+#   Some language servers require additional setup/installations.
+#   See here for details: https://oraios.github.io/serena/01-about/020_programming-languages.html#language-servers
+# When using multiple language servers, the first language server that supports a given file will be used for that file.
+# The first language server is the default language and the respective language server will be used as a fallback.
+# Note that when using the JetBrains backend, language servers are not used and this list is correspondingly ignored.
+language_servers:
+- bash
+
+# the encoding used by text files in the project
+# For a list of possible encodings, see https://docs.python.org/3.11/library/codecs.html#standard-encodings
+encoding: "utf-8"
+
+# optional shell command to run before the language backend (LSP or JetBrains) is initialised.
+# the command runs in the project root directory and is only executed if the project is trusted
+# (see trusted_project_path_patterns in the global configuration).
+# serena waits for the command to exit: a non-zero exit code is logged as an error but does not
+# abort activation. a per-project timeout (activation_command_timeout, default 180s) is the safety
+# backstop for non-terminating commands; on expiry the process is killed and activation continues.
+# example: activation_command: "npx nx run-many -t build"
+activation_command:
+
+# maximum time in seconds to wait for activation_command to complete before killing it (default 180s).
+# must be a positive number.
+activation_command_timeout: 180.0
+
+# line ending convention to use when writing source files.
+# Possible values: unset (use global setting), "lf", "crlf", or "native" (platform default)
+# This does not affect Serena's own files (e.g. memories and configuration files), which always use native line endings.
+line_ending:
+
+# The language backend to use for this project.
+# If not set, the global setting from serena_config.yml is used.
+# Valid values: LSP, JetBrains
+# Note: the backend is fixed at startup. If a project with a different backend
+# is activated post-init, an error will be returned.
+language_backend:
+
+# The interface through which the agent (LLM) accesses Serena's functionality (overrides the global setting).
+# Valid values: tools, REPL (see the global configuration for details); leave empty to use the global setting.
+# Note: the interface is fixed at startup. If a project is activated post-init, its setting is not applied.
+agent_interface:
+
+# whether to use project's .gitignore files to ignore files
+ignore_all_files_in_gitignore: true
+
+# advanced configuration option allowing to configure language server-specific options.
+# Maps the language key to the options.
+# The settings are considered only if the project is trusted (see global configuration to define trusted projects).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#language-server-specific-settings
+ls_specific_settings: {}
+
+# list of workspace folder paths (LSP backend only).
+# These folders will be used to build up Serena's symbol index.
+# Paths must be within the project root and should thus be relative to the project root.
+# Furthermore, the paths should not be filtered by ignore settings.
+# Default setting: The entire project root folder (".") is considered.
+# In (large) monorepos, this can be used to index only subfolders of the project root, e.g.
+#   ls_workspace_folders:
+#     - "./subproject1"
+#     - "./subproject2"
+ls_workspace_folders:
+- "."
+
+# list of additional workspace folder paths for cross-package reference support.
+# Paths can be absolute or relative to the project root.
+# Each folder is registered as an LSP workspace folder, enabling language servers to discover
+# symbols and references across package boundaries, but these folders are not indexed by Serena,
+# i.e. the respective symbols will not be found using Serena's symbol search tools.
+# Example:
+#   additional_workspace_folders:
+#     - ../sibling-package
+#     - ../shared-lib
+ls_additional_workspace_folders: []
+
+# list of additional paths to ignore in this project.
+# Same syntax as gitignore, so you can use * and **.
+# Important: quote patterns that start with `*`, otherwise YAML treats them as aliases.
+# Example:
+#   ignored_paths:
+#     - "examples/**"
+#     - ".worktrees/**"
+#     - "**/bin/**"
+#     - "**/obj/**"
+# Note: global ignored_paths from serena_config.yml are also applied additively.
+ignored_paths: []
+
+# whether the project is in read-only mode
+# If set to true, all editing tools will be disabled and attempts to use them will result in an error
+# Added on 2025-04-18
+read_only: false
+
+# list of tool names to exclude.
+# This extends the existing exclusions (e.g. from the global configuration)
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+excluded_tools: []
+
+# list of tools to include that would otherwise be disabled (particularly optional tools that are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+included_optional_tools: []
+
+# fixed set of tools to use as the base tool set (if non-empty), replacing Serena's default set of tools.
+# This cannot be combined with non-empty excluded_tools or included_optional_tools.
+# Find the list of tools here: https://oraios.github.io/serena/01-about/035_tools.html
+fixed_tools: []
+
+# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.find_symbol") to exclude from the REPL.
+# This extends the existing exclusions (e.g. from the global configuration).
+excluded_apis: []
+
+# list of APIs (facades or facade methods, e.g. "lsp" or "lsp.get_diagnostics_for_symbol") to include in the REPL
+# that would otherwise be disabled (particularly optional methods, which are disabled by default).
+# This extends the existing inclusions (e.g. from the global configuration).
+included_apis: []
+
+# list of mode names that are to be activated by default, overriding the setting in the global configuration.
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# If the setting is undefined/empty, the default_modes from the global configuration (serena_config.yml) apply.
+# Otherwise, this overrides the setting from the global configuration (serena_config.yml).
+# Therefore, you can set this to [] if you do not want the default modes defined in the global config to apply
+# for this project.
+# This setting can, in turn, be overridden by CLI parameters (--mode).
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+default_modes:
+
+# list of mode names to be activated additionally for this project, e.g. ["query-projects"]
+# The full set of modes to be activated is base_modes (from global config) + default_modes + added_modes.
+# See https://oraios.github.io/serena/02-usage/050_configuration.html#modes
+added_modes:
+
+# initial prompt for the project. It will always be given to the LLM upon activating the project
+# (contrary to the memories, which are loaded on demand).
+initial_prompt: ""
+
+# time budget (seconds) per tool call for the retrieval of additional symbol information
+# such as docstrings or parameter information.
+# This overrides the corresponding setting in the global configuration; see the documentation there.
+# If null or missing, use the setting from the global configuration.
+symbol_info_budget:
+
+# list of regex patterns which, when matched, mark a memory entry as read‑only.
+# Extends the list from the global configuration, merging the two lists.
+read_only_memory_patterns: []
+
+# list of regex patterns for memories to completely ignore.
+# Matching memories will not appear in list_memories or activate_project output
+# and cannot be accessed via read_memory or write_memory.
+# To access ignored memory files, use the read_file tool on the raw file path.
+# Extends the list from the global configuration, merging the two lists.
+# Example: ["_archive/.*", "_episodes/.*"]
+ignored_memory_patterns: []

docs/research/README.md

@@ -29,6 +29,7 @@ Each file here is a **dated snapshot**: research or a decision record as it stoo
 | [proxy-shadow-pricing.md](proxy-shadow-pricing.md) | Superseded by the OmniRoute migration | Shadow cost was wired into LiteLLM config, and nothing implements it now |
 | [proxy-tool-choice.md](proxy-tool-choice.md) | Superseded by the OmniRoute migration | Chose LiteLLM. It mentions Open WebUI, which has been removed |
 | [inference-engine-spec-decoding-multi-gpu.md](inference-engine-spec-decoding-multi-gpu.md) | Current | llama.cpp vs. vLLM vs. SGLang on R9700: drafters, multi-GPU, licenses, `--cache-ram` misspelling bug. Not benchmarked |
+| [qwen3.8-27b-decode-speed.md](qwen3.8-27b-decode-speed.md) | Current | Decode tokens/s: why DFlash2 acceptance and speed fall with depth (draft length is 5, not 7), n_max/p_min, KV and slot costs, bandwidth ceiling, ordered experiment plan. Builds on the throughput-tuning note. Proposals only, nothing applied |
 | [qwen3.8-27b-quant.md](qwen3.8-27b-quant.md) | Current | Main model quant and VRAM budget |
 | [qwen3.8-27b-throughput-tuning.md](qwen3.8-27b-throughput-tuning.md) | Current | Tokens/s tuning from the server log, DDR RAM as a cache tier, comparison with five outside R9700 configs. Proposals only, nothing applied |
 | [qwen3.8-27b-tool-calling.md](qwen3.8-27b-tool-calling.md) | Current | Tool-calling parser risk for the main model |

docs/research/qwen3.8-27b-decode-speed.md

@@ -0,0 +1,226 @@
+# Research: decode tokens/s on the R9700 (llama-server, Qwen3.8-27B + DFlash2)
+
+**Date:** 2026-10-20
+
+**Question:** Why is decode at ~52k context about plain-decode speed (26-28 t/s) when an older, shorter-context log averaged 45.9 t/s, and what can be changed to raise decode (and prefill where relevant)? Evidence: `llmserver.log` (re-analysed per request) plus llama.cpp source.
+
+**Scope note:** research only. Nothing in compose or `.env.example` was changed and nothing was deployed. This note builds on [`qwen3.8-27b-throughput-tuning.md`](qwen3.8-27b-throughput-tuning.md) (cache/prefill ranking, DDR tier, five outside configs) and [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) (quant choice, VRAM budget). It does not repeat them, and it **corrects one premise of the first** (see finding 1).
+
+**Labels.** **[V]** verified from source (URL pinned to a commit, or a repo file:line). **[L]** computed from `llmserver.log` by me. **[R]** reported by a third party (a page or thread, not re-run). **[I]** inferred, not measured. WebFetch summaries of web pages were produced by a small model, so every number taken from a page is **[R]** even when the page is first-party.
+
+**Source pin.** llama.cpp `master` at `839a572a4cdecdb80d211297b8e7d7323a6e99d2` (fetched 2026-10-20). Links below point at that commit. The running image (`server-rocm`) may be older than this commit. Its build string is not in the logs I have, so every "[V]" is "true on that commit".
+
+## TL;DR
+
+| # | Finding | Label |
+|---|---|---|
+| 1 | The draft length has been **5, not 7**, in the old log too. All 306 `draft acceptance` summaries have `generated` divisible by 5 and `mean len = 1 + accepted / (generated / 5)`. So `n_max=5` in the new startup log is the same setting, not a new clamp. The earlier note's "n_max is 7" came from compose and was wrong | [L] + [V] |
+| 2 | master clamps `n_max` to `block_size - 1` (= 7 for block_size 8). 5 is below that, so no clamp fired. 5 must have been **requested**: most likely `LLAMA_SPEC_N_MAX=5` in the server's `.env` (the repo `.env.example` default differs, and `.env` is not in the repo) | [V] + [I] |
+| 3 | Decode speed tracks acceptance almost linearly (r = 0.97 over 256 requests) and acceptance falls with depth: 0.75 at <8k, 0.57 at 8-20k, 0.42 at 20-40k, 0.40 at 40-60k, while decode falls 64.6 to 31.9 t/s. The 45.9 t/s average was dominated by short-context requests | [L] |
+| 4 | Bandwidth ceiling for plain decode: 17.6 GB / 640 GB/s = 27.5 ms = **36.4 t/s** (33.0 t/s at 52k depth with q8_0 KV). At the 79-81 % efficiency best seen for dense models on this card that is 26-27 t/s, which is exactly the measured "plain" rate | [I] on [R] inputs |
+| 5 | A spec step costs 74-94 ms in the old log, against a ~28-30 ms weight-streaming floor. About 45 ms per step is fixed overhead (drafter, 6-wide verify, sync, sampling), so speculation only pays when acceptance is high | [L] + [I] |
+| 6 | DFlash2 draws its draft greedily and gives the server no draft distribution, so at temp > 0 a draft token is accepted only if the target's own sampled token equals it. Sampler settings on the client therefore change acceptance directly | [V] |
+| 7 | `GPU_MAX_HW_QUEUES=1` is free for default llama.cpp: multi-stream graph execution is opt-in (`GGML_CUDA_GRAPH_OPT=1`) | [V] |
+
+Top 3 experiments (details in "Experiment plan"): **A** sweep `--spec-draft-n-max` 3, 4, 5, 7 and add `--spec-draft-p-min`; **B** measure acceptance versus client sampler settings with a fixed prompt (temp, top-p/top-k, penalties); **C** decompose the 74-94 ms step with `llama-bench` (pp6 at depth vs tg1) and `rocm-smi` clocks, before touching KV type, slots or quant.
+
+## 1. DFlash2: why acceptance and speedup vary
+
+### 1.1 The n_max 5 vs 7 discrepancy
+
+- The init line prints `n_max=%d` from `this->params.n_max` **before** the clamp: [`common/speculative.cpp#L1086`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1086). The clamp is [`#L1090-L1099`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1090): `n_draft_max = (is_dspark && sample_from_anchor) ? block_size : block_size - 1`, with the warning `requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d`. **[V]**
+- For this drafter (`block_size=8`, DFlash2 is not DSpark) the limit is 7. So a requested 7 passes unclamped and a requested 5 prints 5. The init line therefore shows the **requested** value; no clamp explains 5. **[V]** (on that commit)
+- `n_extract=5` in the log is `target_layer_ids_n`, the number of target layers whose features feed the drafter ([`#L1040-L1044`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1040)). It is not used in the clamp, so the 5 = 5 match is a coincidence on this commit. **[V]** An older build could clamp differently; I could not check history for this. **[I]**
+- Log proof that 5 is the real draft length. The server prints `mean len = 1 + accepted / verif_steps` ([`tools/server/server-context.cpp#L704-L725`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L704)). With `verif_steps = generated / 5` this reproduces the printed mean len in all 306 summaries; with 7 it matches none. Examples: `llmserver.log:3` (47 accepted / 105 generated, 21 steps, 47/21 + 1 = 3.24), `:36` (64/170, 34 steps, 2.88), `:114` (828/2050, 410 steps, 3.02). **[L]**
+- To confirm on the server: `grep -n SPEC_N_MAX .env` and `docker logs llama-server | grep -E "n_max|exceeds the trained"`. **[I]**, not run.
+
+### 1.2 What the drafter does per step
+
+- The drafter input is `[id_last, <mask> * n_max]` and one forward pass denoises the whole block. DFlash2 then greedily walks a candidate lattice (`selector_top_k` candidates per position, argmax per step): [`#L1296-L1310`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1296) and [`#L1336-L1366`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1336). The block size in the drafter batch is `n_max + 1`, so a smaller `n_max` shrinks the drafter batch too. **[V]**
+- `--spec-draft-p-min` on DFlash2: stop extending the draft at the first position where softmax(selector scores) at the argmax is below `p_min` ([`#L1349-L1358`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1349)). It is a per-position stop, default 0.00 ([`common/common.h#L333`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L333), [`common/arg.cpp#L4248`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4248)). The git log shows DFlash2-specific p_min arrived with the DFlash2 PR ([#27342](https://github.com/ggml-org/llama.cpp/pull/27342), commit titles "Add p_min in DFlash2" per the PR page **[R]**). **[V]** for the code. The `generated` multiples of 5 in the log confirm p_min was 0 in the old run. **[L]**
+- `n_min` (`--spec-draft-n-min`, default 0): a draft shorter than `n_min` is discarded ([`#L1362-L1364`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1362)). **[V]**
+- `sample_from_anchor=true` only changes the layout for DSpark drafters; for DFlash2 the block is `n_max + 1` ([`#L1298`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1298)). **[V]**
+- `--spec-draft-sampling probabilistic` ([`arg.cpp#L4264`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4264)) is documented as sampling the draft and verifying by rejection. The DFlash2 branch never fills the draft distribution (`result_q`), and the server then falls back to plain sample-and-match: `use_rejection = use_spec_rejection() && !spec_draft_q.empty()` ([`server-context.cpp#L4537`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L4537)). So that flag does nothing for DFlash2. **[V]** (read of the code path; not run).
+
+### 1.3 Why acceptance varies
+
+1. **Sampling.** With the DFlash2 drafter, a draft token is accepted iff the target's sampled token equals it ([`common/sampling.cpp#L690-L720`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/sampling.cpp#L690), `draft[i] != id` breaks). At temp 0 the target token is its argmax, the best case for matching a greedy draft. At temp > 0 it falls with flatness of the target distribution and with anything that reshapes it (top-p/top-k, repetition or presence penalties). **[V]** for the mechanism. The size of the effect for this workload is unmeasured. **[I]**
+   - Contradiction to resolve: the earlier note cites a third-party claim that greedy sampling "collapses" DFlash2 acceptance. The source reading says the opposite for the server path. Treat that claim as unverified until experiment B settles it.
+   - Server-side the temp > 0 test is `task->params.sampling.temp > 0` ([`server-context.cpp#L508`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L508)). The drafter itself ignores temperature. **[V]**
+2. **Context depth.** In the old log, acceptance halves from <8k to >40k tokens (0.75 to 0.40, table below). A PR commenter also reported code prompts decaying faster with context length than prose ([PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) comments) **[R]**. The model card gives no context length for the drafter. **[R]**
+3. **Content type.** Not separable in this log (no prompt/response classes). The authors' acceptance lengths (mean tokens per verify step, 7 drafted, block size 8, temp 1.0 / top-p 0.95 / top-k 20, SGLang on an H200) are GSM8K 5.46, MATH-500 5.28, HumanEval 4.39, MBPP 4.79 ([z-lab/Qwen3.8-27B-DFlash2](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2), via a search summary) **[R]**. Your mean len of 2.9-4.9 (max possible 6 at n_max 5) is below that, partly because those are short prompts. **[I]**
+4. **Concurrency.** Issue [#27117](https://github.com/ggml-org/llama.cpp/issues/27117) (gfx1151, closed "not planned"): acceptance collapses as the number of concurrently drafting sequences grows (0.51-0.66 at 4 active, 0.06-0.49 at 16), and `--spec-draft-n-max 1` recovered it. **[R]** Your two slots are far below that regime, but the bimodal per-slot acceptance in that issue is worth watching if both slots decode at once. **[I]**
+
+### 1.4 Authors' recommended values
+
+- Model card quick start: `--spec-type draft-dflash --spec-draft-n-max 7` ([z-lab/Qwen3.8-27B-DFlash2-GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF)) **[R]**. No block size, temperature or p_min advice is given there. The card says the output is lossless (greedy identical, sampling distribution preserved).
+- A PR commenter on an RTX 3090 used `--spec-draft-n-max 4` and reported that 7 was slower: DFlash2 at n=4 averaged 62.21 t/s at 0.6153 acceptance, MTP 57.79 t/s at 0.6317 ([PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) comments) **[R]**.
+- Drafter quant does not matter for acceptance: acceptance length 5.28 (BF16), 5.13 (Q8_0), 5.39 (Q4_K_M) on 8 GSM8K prompts; the card itself implies Q4_K_M above BF16 is noise. Q4_K_M is ~1.1 GB, so it also costs the least bandwidth. **[R]**
+
+### 1.5 What the old log shows (re-analysis)
+
+Per-request join of `stop processing: n_tokens` (final depth), `eval time` t/s and `draft acceptance`, requests with >= 200 generated tokens, 256 requests (script: ad-hoc Python over `llmserver.log`, not committed). Draft length 5, so tokens per verify step is 1 + 5 x acceptance.
+
+| Final depth | n | Decode t/s (mean) | Acceptance (mean) | Tokens/step | Step time (tokens/step / t/s) |
+|---|---|---|---|---|---|
+| < 8k | 125 | 64.6 | 0.75 | 4.75 | ~74 ms |
+| 8-20k | 28 | 44.2 | 0.57 | 3.85 | ~87 ms |
+| 20-40k | 17 | 34.6 | 0.42 | 3.10 | ~90 ms |
+| 40-60k | 86 | 31.9 | 0.40 | 3.00 | ~94 ms |
+
+- Correlation of per-request decode t/s with acceptance: r = 0.97, slope ~90 t/s per 1.0 acceptance. **[L]**
+- Step time rises only ~20 ms from 0 to 50k depth, while tokens per step drops 4.75 to 3.0. So about **80 % of the decode loss with depth is acceptance, not attention cost**. **[L]** (derived, mean-of-ratios caveat).
+- The new run (26-28 t/s at ~52k, tg_3s 20-33) is at or just below the 40-60k bucket (31.9), consistent with the same acceptance regime. **[L]**
+
+## 2. Decode speed versus context depth
+
+### 2.1 What the attention costs here
+
+- Only 16 of 64 layers keep a KV cache ([`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) "Model identity"). KV bytes per token: 16 layers x 2 x 4 heads x 256 x element size = 65,536 B (f16), 34,816 B (q8_0, 1.0625 B/element), 18,432 B (q4_0, 0.5625 B/element). At 52k tokens: 3.4 GB (f16), 1.81 GB (q8_0), 0.96 GB (q4_0). Against 17.6 GB of weights read per token, q8_0 KV adds ~10 % bytes at 52k, f16 ~19 %, q4_0 ~5 %. **[I]** (arithmetic from the quant note's architecture facts.)
+- So KV precision can move plain decode at 52k by at most a few percent. Going to q4_0 would buy ~1 GB less read (~5 %) at an unmeasured quality cost. Do not expect it to fix the 2x gap. **[I]**
+
+### 2.2 Which flash-attention kernel runs on gfx1201
+
+From [`ggml/src/ggml-cuda/fattn.cu#L540-L722`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L540) (`ggml_cuda_get_best_fattn_kernel`), applied to head_dim 256, 24 query heads, 4 KV heads (gqa_ratio 6, effective ratio 2), K/V q8_0:
+
+| Query batch (tokens) | Kernel on gfx1201 | Why (source) |
+|---|---|---|
+| 1 (plain decode) | VEC | WMMA needs `Q->ne[1] * gqa_ratio_eff > 16` for head_dim > 128 ([`#L698-L702`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L698)), so 1 x 2 fails. For quantized K/V the vector kernel is chosen when `Q->ne[1] <= 2` ([`#L714-L720`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L714)). Reads q8_0 directly. |
+| 2 | VEC (quantized) | same rule |
+| 3 to 8 (verify batch of a spec step) | **TILE** | 8 x 2 = 16 is not > 16, so no WMMA; quantized KV is not eligible for VEC above 2 tokens. |
+
+- TILE and MMA kernels "need f16 K/V" ([`#L735-L740`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn.cu#L735)): with a quantized cache the K and V views are converted to f16 scratch before the kernel runs. So a verify step at 52k converts ~100M q8_0 elements per layer (K plus V) to f16 on 16 layers. Rough extra traffic: read 1.8 GB plus write 3.4 GB plus read 3.4 GB per step, ~8.6 GB, ~13 ms at 640 GB/s. **[I]** (the conversion and kernel choice are [V]; the traffic estimate and its cost are inferred, not profiled). That is consistent with the ~20 ms step-time rise from 0 to 50k in 1.5. With f16 KV the conversion disappears (TILE reads f16 directly) but the cache is 1.9x larger, so the net is not obvious. **[I]**
+- No kernel-level bug report for gfx1201 + quantized KV turned up in the sources I read. The only RDNA4 data point is [R]: on RADV (Vulkan) q8_0 vs q4_0 KV was within +-0.5 % ([discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043)). That is a different backend and says nothing about the HIP path.
+
+### 2.3 `--parallel 2` and `--kv-unified`
+
+- With `--kv-unified` there is one KV stream. `get_n_kv` pads `cells.used_max_p1()`, the highest occupied cell across **all** sequences ([`src/llama-kv-cache.cpp#L1282-L1292`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-kv-cache.cpp#L1282)). The flash-attention kernels only skip fully masked tiles at the **end** of the range (`flash_attn_mask_to_KV_max`, [`fattn-common.cuh#L671-L723`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/fattn-common.cuh#L671)). So when the other slot holds a long prompt in higher cells, slot 0's decode may read and mask-skip past tens of thousands of foreign cells. **[V]** for both pieces, **[I]** for the combined cost (not profiled; which cells each slot occupies is not logged).
+- This matches the pattern in the older log (requests alternate slot 0 / slot 1, each ~40-50k tokens, [throughput note](qwen3.8-27b-throughput-tuning.md) "Pattern at `:380`-`:470`"), but no measurement separates it. **Experiment D** below tests it.
+- `--kv-unified-per-slot N` exists ([`arg.cpp#L1652`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L1652)) and `--no-kv-unified` gives one stream per slot ([`arg.cpp#L1727-L1734`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L1727)). Non-unified mode splits `--ctx-size` evenly, so one slot could no longer use more than half the context. **[V]**
+
+### 2.4 Checkpoints and recurrent state during decode
+
+- The target is `qwen35` architecture, which `llm_arch_supports_rs_rollback` lists ([`src/llama-arch.cpp#L1256-L1262`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-arch.cpp#L1256)). I am assuming Qwen3.8 loads as that arch; the name was not verified. **[I]**
+- For DFlash/DSpark the server requests `n_rs_seq = draft.n_max` recurrent-state snapshots per sequence ([`common/common.h#L401-L407`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L401); [`src/llama-context.cpp#L191-L195`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/src/llama-context.cpp#L191)). The server creates a **speculative checkpoint** only if `draft.size() > llama_n_rs_seq(ctx_tgt)` ([`server-context.cpp#L3541-L3558`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L3541)), and restores one only if `n_rollback > n_rs_seq` ([`#L4559`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L4559)). With `n_rs_seq == n_max` that never happens, so a rejection is a snapshot-index rollback and not a state copy. **[V]**
+- If the build is older or the arch is not on the rollback list, the `COMMON_CONTEXT_SEQ_RM_TYPE_FULL` path copies a checkpoint every step (`use_ckpt_tgt`, [`#L3463`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L3463)); the startup line `speculative decoding will use checkpoints` ([`#L1388-L1390`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L1388)) would show it. Check the startup log for that line. **[V]** for the code, **[I]** for which case applies.
+- `--ctx-checkpoints` / `--checkpoint-min-step` (defaults 32 and 8192, [`common.h#L639-L642`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L639)) are for prompt/prefix reuse. Nothing I read ties them to the decode loop, so they should not cost decode time. **[I]**
+- The "cache_reuse is not supported by this context" startup line is the server's `llama_memory_can_shift` check ([`server-context.cpp#L1328-L1336`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/tools/server/server-context.cpp#L1328)). It confirms the open question in the throughput note. **[V]**
+
+## 3. Backend, HIP flags, clocks
+
+### 3.1 ROCm versus Vulkan (RADV)
+
+Everything here is **[R]** unless noted; the thread posts are self-reported and mixed in setup.
+
+| Source | Setup | Numbers |
+|---|---|---|
+| [discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043) (JohnTDI-cpu, build 8554) | R9700, RADV, Qwen3.5-27B Q4_K_M | tg128 29.07 t/s (RADV), 29.01 (AMDVLK); pp512 798 (RADV). Computed there as ~456 GB/s, 71 % of 640 GB/s; "best dense ~79-81 %" |
+| same | R9700, ROCm, 27B Q8_0, q4_0 KV, 64k ctx | pp 960 (3.7k) to 708 (58.7k) t/s; tg 42.8 to 28.0 t/s over the same range (MTP use not stated, and 42.8 t/s is above a 29 GiB Q8_0 plain-decode ceiling of ~22 t/s, so it is almost certainly with speculation) |
+| same | Windows, HIP, gfx1201 | pp512 1106-1211; tg128 29.07; "Vulkan decode still wins" |
+| same | R9700, Qwen3.6-27B UD-Q4_K_XL, q4_0 KV, MTP (build b9203) | tg128 28.2 baseline, 47.4 with `mtp-3` |
+| [discussion #20881](https://github.com/ggml-org/llama.cpp/discussions/20881) | R9700 and others | MoE Qwen3-Coder-30B-A3B Q4_0 ~196 t/s on RADV; one ROCm segfault report on Qwen3.5-35B-A3B |
+| [throughput note](qwen3.8-27b-throughput-tuning.md) (cat5edopeHA) | R9700, MTP n3 | Vulkan 68.5 / ROCm 55.1 short, 50.5 / 40.2 at 32k, 40.9 / 31.2 at 64k |
+
+- A search summary also cites a blog (not fetched) with ROCm ahead for dense 27B decode (42.8 vs 29.1), the opposite of cat5edopeHA. The sources conflict, so only a local A/B decides. **[R]**
+- DFlash2 on Vulkan: not verified. The DFlash graph and feature extraction live in common code and the model graph, so it may work, but I did not check that a Vulkan build runs `draft-dflash`. **[I]**
+- Prefill: HIP uses MMQ on RDNA4 unconditionally (`ggml_cuda_should_use_mmq` returns true for RDNA4, [`mmq.cu#L468-L471`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmq.cu#L468)), so `GGML_CUDA_FORCE_MMQ` changes nothing on gfx1201. **[V]**
+
+### 3.2 Decode kernels for the verify batch
+
+- Weights are Q4_K-family. Matrix multiplies with up to 8 columns go to MMVQ (`MMVQ_MAX_BATCH_SIZE 8`, [`mmvq.cuh`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmvq.cuh)). RDNA4 has no per-type override, so `ne11 <= 8` uses MMVQ ([`mmvq.cu#L318-L430`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/mmvq.cu#L318)). The source comment says k-quants "cost more to decode and mvq redoes that per column". **[V]** So the verify batch of 6 (n_max 5) or 8 (n_max 7) pays dequant ALU work per column; cost grows with batch width even though weight bytes are read once. **[I]** (the per-column cost on gfx1201 was not measured). This is the mechanism behind "lower n_max can win": it shrinks both the MMVQ columns and the drafter batch.
+- Batch 9 or more would fall to MMQ, which is not reachable: `n_max` is capped at 7 so the batch is at most 8. **[V]**
+
+### 3.3 HIP flags and environment variables
+
+| Item | Verdict | Source |
+|---|---|---|
+| `GGML_HIP_GRAPHS` | Already ON in the CMake default and the official ROCm Dockerfile does not turn it off. Variable graph shapes force a re-warmup: any change in node properties resets `warmup_complete` ([`ggml-cuda.cu#L4806-L4845`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/ggml-cuda.cu#L4806)). With `p_min > 0` the verify batch width varies, so graphs may be re-captured more often. Debug with `GGML_CUDA_GRAPH_DEBUG=1`; disable with `GGML_CUDA_DISABLE_GRAPHS=1`. | [V] ([`ggml/CMakeLists.txt#L217`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/CMakeLists.txt#L217), [`rocm.Dockerfile`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/.devops/rocm.Dockerfile)); effect on speed **[I]** |
+| `GGML_CUDA_FORCE_MMQ` | No effect on RDNA4 (see 3.1) | [V] |
+| `GGML_CUDA_GRAPH_OPT=1` | Opt-in multi-stream concurrent branches ([`ggml-cuda.cu#L5016-L5030`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/ggml/src/ggml-cuda/ggml-cuda.cu#L5016)). Off by default | [V] |
+| `GPU_MAX_HW_QUEUES=1` | HIP caps hardware queues per process per device (default 4, [HIP env docs](https://rocm.docs.amd.com/projects/HIP/en/docs-7.2.3/reference/env_variables.html)). llama.cpp issues all kernels on one stream unless `GGML_CUDA_GRAPH_OPT=1`, so one queue should cost nothing for decode. The repo set it for [ROCm/ROCm#5706](https://github.com/ROCm/ROCm/issues/5706) (`docs/architecture.md#gpu-access`). Do not combine `GGML_CUDA_GRAPH_OPT=1` with it without a test | [R] for the doc, [V] for the llama.cpp side, **[I]** for "no decode cost" |
+| `-bs, --backend-sampling` | Experimental, default off ([`arg.cpp#L2310`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L2310)). Moves target sampling onto the GPU, which would remove the CPU sampling of 6 logit rows per step. Interplay with speculation is unverified | [V] flag exists; benefit **[I]** |
+| `rocm-smi` perf level / compute profile | In #21043: "ROCm `rocm-smi` high vs auto: auto was faster" on a single card; a LACT 330 W cap with COMPUTE profile gave tg 102 to 110 on a dual-card MoE run; `power_dpm_force_performance_level=high` gave +14 % tg on one Vulkan run. Mixed, hardware and backend differ | [R] |
+| PCIe ASPM `pcie_aspm.policy=performance` | Same thread: dense decode on RADV 29.30 to 32.46 (+10.8 %), one user, not reproduced on eGPU or dual-slot. It is a runtime sysfs switch, so it is cheap to try | [R] |
+| MCLK not reaching max under load | Same thread, kernel 6.17, RADV | [R]; check with `rocm-smi` during decode, **[I]** |
+
+### 3.4 Threads, mmap, mlock, poll
+
+- With every layer on the GPU the CPU is only a scheduler. #21043 tested `-t`, `-b`, `-mmp 0`, `--prio 2`, CPU pinning, hugepages and `nice` on RADV and found all within +-0.3 % **[R]**. The server default `--poll` level is 50 ([`common.h#L77`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/common.h#L77)) and `-t` was -1 (16 here). I found no evidence these matter for a GPU-resident run, so do not spend experiments on them. **[I]**
+- One exception worth knowing: speculative drafting has its own thread/poll flags (`--spec-draft-threads`, `--spec-draft-poll`, [`arg.cpp`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp)), but the drafter is on the GPU too, so they should be equally inert. **[I]**
+
+## 4. Quant and model options
+
+Inputs: GGUF sizes from [`qwen3.8-27b-quant.md`](qwen3.8-27b-quant.md) (UD-Q4_K_XL 17.6 GB, UD-Q4_K_M 16.5, UD-Q4_K_S 15.4) and the throughput note (UD-IQ4_XS 13.3 GB). Bandwidth 640 GB/s is the card spec as quoted in #21043 **[R]**. Weights only; 52k q8_0 KV adds 1.81 GB (2.1).
+
+| Quant | Bytes read/token | Ceiling at 640 GB/s | At 80 % efficiency | Same with 52k q8_0 KV (ceiling / 80 %) |
+|---|---|---|---|---|
+| UD-Q4_K_XL (running) | 17.6 GB | 36.4 t/s (27.5 ms) | 29.1 | 33.0 / 26.4 |
+| UD-Q4_K_M | 16.5 GB | 38.8 | 31.0 | 35.0 / 28.0 |
+| UD-Q4_K_S | 15.4 GB | 41.6 | 33.2 | 37.2 / 29.8 |
+| UD-IQ4_XS | 13.3 GB | 48.1 | 38.5 | 43.5 / 34.8 |
+
+- All **[I]** (arithmetic). The 80 % factor is the "best dense ~79-81 %" figure in #21043 **[R]**. The measured plain-decode rate at depth, 26-28 t/s, sits on the 52k column for Q4_K_XL (26.4), so the card is already near its bandwidth limit for plain decode. There is no hidden 2x in the non-speculative path.
+- A smaller quant gains at most ~13 % (Q4_K_M) to ~32 % (IQ4_XS) on the plain-decode ceiling and also shrinks the spec verify weight read. Quality cost is not measured here. The IQ4 family dequantizes slower than Q4_K, so the real gain is below the ceiling ratio. **[I]** Test with KL divergence or perplexity on a small fixed corpus (`llama-perplexity`) before any switch.
+- Drafter quant: Q4_K_M at ~1.1 GB is already the smallest tested, with acceptance as good as BF16 (1.4). Nothing to gain. **[R]**
+- Per spec step the drafter adds ~1.1 GB of reads (~1.7 ms at the ceiling), small against the ~45 ms fixed overhead found in 1.5. **[I]**
+
+## 5. Where the step time goes (open)
+
+Old-log step times are 74 ms at <8k and 94 ms at 40-60k ([1.5](#15-what-the-old-log-shows-re-analysis)). The weight-streaming floor at 80 % efficiency is ~34 ms. The remaining ~40-60 ms per step is not explained by any single item I could verify. Candidate contributors, none measured:
+
+1. 6-column MMVQ with k-quant dequant per column (3.2).
+2. Drafter forward over `n_max + 1` positions plus injecting 5 layers of target features per accepted token (`n_extract=5`, [`speculative.cpp#L1040`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/speculative.cpp#L1040)), including the encoder fused into KV injection ([commit 662a0b0](https://github.com/ggml-org/llama.cpp/commit/662a0b012)).
+3. CPU sampling of 6 full-vocab logit rows plus device-to-host copies per step (`-bs` is off).
+4. Host synchronisation between drafter and target per step (two contexts, no overlap).
+5. Quantized-to-f16 KV conversion for the TILE kernel at depth (2.2).
+
+Experiment C isolates 1 and 5 without the server; the rest need a profile (`rocprofv3`), which is out of scope here.
+
+## 6. Experiment plan
+
+Rules: one change per run, cold slot, same prompt, restart the server between configs with `./scripts/update.sh` on the server (this box cannot deploy). All numbers come from the server log lines `prompt eval time`, `eval time` and `draft acceptance` (`llmserver.log:32-36` show the format).
+
+### Benchmark protocol (minimal, comparable)
+
+1. **Fixed prompts** (kept in the repo's scratch area, not committed): P-code = ~8k tokens of source plus the instruction "continue the file" and P-code-52k = the same material padded to ~52k tokens; P-reason = a reasoning question over the same 52k context. Same bytes every run.
+2. **Fixed request:** `n_predict=512` (400+ tokens so the 3-s `tg` window averages out), `seed` fixed, `cache_prompt` false for the first request so prefill is cold, then the same request again with the prompt cache warm only if you want decode in isolation.
+3. **Sampler:** run each prompt twice, once with the production client settings (record them from the request body) and once with `temperature=0`. Record the sampler fields in the run name.
+4. **Repeats:** N = 5 per cell, both slots idle except the request under test; discard the first.
+5. **Read** from the log for each request: prompt tokens/s (`prompt eval time`), decode t/s (`eval time`), `draft acceptance` and `mean len`. Report median and min/max, not the mean.
+6. **Decode-only control:** `llama-bench` inside a throwaway container (stop `llama-server` first, one GPU): `llama-bench -m <model> -ngl 99 -fa 1 -ctk q8_0 -ctv q8_0 -p 0 -n 256 -d 0,16384,52000 -r 3`. `-n 256` avoids the "implicit ubatch 128 on hybrid models" artifact reported with `-n 128` in #21043 **[R]**. `llama-bench` does not use speculation or the prompt cache.
+7. **Environment capture** per run: `rocm-smi --showclocks --showpower --showtemp` once during the decode (SCLK, MCLK), plus the full startup log (build string, `n_rs_seq`, checkpoint line).
+
+### Experiments, in priority order
+
+| # | Change (exact) | Expected effect | How to measure | Risk |
+|---|---|---|---|---|
+| A | `LLAMA_SPEC_N_MAX` = 3, 4, 5, 7 (one run each); then 5 with `--spec-draft-p-min 0.3` and `0.5` | Fewer wasted drafted tokens and a narrower verify batch when acceptance is ~0.4; could be neutral or negative when acceptance is high. Direction at 52k is the point; short context favors the larger value. No estimate of size without data | Decode t/s, `mean len`, `generated` (must be a multiple of the n_max, or not if p_min is on), on P-code and P-code-52k | Low. `p_min` makes batch widths vary, which can trigger graph re-capture (3.3); if so, compare with `GGML_CUDA_DISABLE_GRAPHS=1` |
+| B | No server change: send P-code-52k / P-reason with (1) production sampler, (2) `temperature=0`, (3) production minus penalties, (4) production minus top-p/top-k | Isolates how much acceptance the sampler costs (6, 1.3). A large gap says the fix is in the client/gateway sampler defaults, not llama.cpp | `draft acceptance` and `mean len` per cell; the full per-position `acc per pos` appears at trace level only | None |
+| C | Stop `llama-server`; in a container with the same image: `llama-bench ... -p 1,2,4,6,8 -n 0 -d 0,52000` and `-p 0 -n 256 -d 0,52000`; rocm-smi clocks during a server decode | Splits step time into verify-batch cost (pp6 time at depth) vs plain tg1. Shows whether verify at depth already costs ~60+ ms | `llama-bench` t/s table; compare pp6 time with measured step time | Low. Needs a short outage of the only GPU consumer |
+| D | `--parallel 1` (then `--no-kv-unified` with `--parallel 2`) on the same P-code-52k | Tells whether the second slot or the unified cache costs single-stream decode (2.3). If decode at 52k jumps, slot layout is the cost | Decode t/s on a request with the other slot idle; then with the other slot holding a 50k context | Medium. `--parallel 1` serialises agents; non-unified halves per-slot max context |
+| E | `echo performance \| sudo tee /sys/module/pcie_aspm/parameters/policy`, and check MCLK/SCLK under load; try `rocm-smi --setperflevel high` only if MCLK sags | Reported +10.8 % on one RADV dense run; unknown for ROCm. Free to try and revert | Decode t/s on P-code (plain control from C) | Low; revert with the previous policy value |
+| F | `-bs` (`--backend-sampling`) | Removes CPU sampling of the verify rows if that is a real share of the step | A/B on P-code | Medium: experimental, speculation interplay unverified |
+| G | `GGML_CUDA_DISABLE_GRAPHS=1` and, separately, `GGML_CUDA_GRAPH_DEBUG=1` to see re-warmup counts | Tells whether graph churn from the variable shapes is a cost | A/B on P-code | Low |
+| H | `-ctk q4_0 -ctv q4_0` at 52k | Saves ~0.85 GB read per token (~4-5 % on plain decode) and removes some f16 conversion traffic; quality risk | A/B on P-code-52k plus a spot-check of output quality | Medium (quality) |
+| I | `-ctk f16 -ctv f16 --ctx-size 131072` | Removes q8_0 to f16 conversion for the TILE verify kernel (2.2); cache is 1.9x larger. VRAM: f16 KV for 131072 tokens is ~8 GB | A/B on P-code-52k | Medium (halves context, VRAM) |
+| J | Smaller quant (UD-Q4_K_M or UD-IQ4_XS) from the same repo | Up to +13 % / +32 % on the plain ceiling (4); quality unmeasured | A/B plus KLD/perplexity | Medium (quality, download) |
+| K | Vulkan (RADV) image, if `draft-dflash` runs on it | Reported decode lead on some setups, but conflicting sources and prefill lag at short context | A/B with the same protocol | High (new image, unverified DFlash2 support) |
+| L | `--spec-type draft-dflash,ngram-mod` (list is accepted, [`arg.cpp#L4314-L4322`](https://github.com/ggml-org/llama.cpp/blob/839a572a4cdecdb80d211297b8e7d7323a6e99d2/common/arg.cpp#L4314)) | Extra drafts on repeated text (tool JSON, code). Interaction with the DFlash drafter is unchecked | A/B on P-code | Medium |
+
+Not worth an experiment: `-t`, `--poll`, `--no-mmap`, `--mlock`, `GGML_CUDA_FORCE_MMQ`, drafter quant (3.4, 3.1, 1.4).
+
+## Open questions
+
+- What does the server `.env` say for `LLAMA_SPEC_N_MAX`, and what does the new startup log print for `speculative decoding will use checkpoints` and `n_rs_seq`?
+- What temperature, top-p/top-k and penalties do OmniRoute clients actually send? If they differ per client, acceptance will differ per client.
+- Is the ~45 ms fixed overhead per spec step in the target verify, the drafter, or the CPU? Experiment C and a `rocprofv3` trace would answer it.
+- Does a Vulkan build run `draft-dflash` at all?
+- Which exact llama.cpp build is in the image, and is it before or after the `n_rs_seq` snapshot change?
+
+## Sources
+
+- Repo: `llmserver.log` (lines cited inline), `docker-compose.yml`, `.env.example`, `docs/architecture.md#llama-server`, `docs/research/qwen3.8-27b-throughput-tuning.md`, `docs/research/qwen3.8-27b-quant.md`.
+- llama.cpp at `839a572a4cdecdb80d211297b8e7d7323a6e99d2`: `common/speculative.cpp`, `common/arg.cpp`, `common/common.h`, `common/sampling.cpp`, `tools/server/server-context.cpp`, `src/llama-kv-cache.cpp`, `src/llama-arch.cpp`, `src/llama-context.cpp`, `src/llama-memory-recurrent.cpp`, `ggml/src/ggml-cuda/{fattn.cu,fattn-common.cuh,mmq.cu,mmvq.cu,mmvq.cuh,ggml-cuda.cu}`, `ggml/CMakeLists.txt`, `.devops/rocm.Dockerfile`. Line numbers were read from `raw.githubusercontent.com/.../master/...` on the day of fetch; if the link anchors drift, search by the quoted identifier.
+- [PR #27342](https://github.com/ggml-org/llama.cpp/pull/27342) (DFlash2), [issue #27117](https://github.com/ggml-org/llama.cpp/issues/27117), [discussion #21043](https://github.com/ggml-org/llama.cpp/discussions/21043), [discussion #20881](https://github.com/ggml-org/llama.cpp/discussions/20881).
+- [z-lab/Qwen3.8-27B-DFlash2-GGUF](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2-GGUF), [z-lab/Qwen3.8-27B-DFlash2](https://huggingface.co/z-lab/Qwen3.8-27B-DFlash2) (via a search summary), [HIP environment variables](https://rocm.docs.amd.com/projects/HIP/en/docs-7.2.3/reference/env_variables.html).
+- Not fetched: the Phoronix R9700 ROCm vs Vulkan review (HTTP 403), AMD ROCm RDNA4 pages beyond the HIP env page.