diff --git a/.env.example b/.env.example index 014c0d1..619dbc4 100644 --- a/.env.example +++ b/.env.example @@ -94,8 +94,16 @@ OMNIROUTE_WS_BRIDGE_SECRET= # Resolved automatically from the host by ./scripts/update.sh — leave blank. COMFYUI_PUID= COMFYUI_PGID= -COMFYUI_VIDEO_GID= -COMFYUI_RENDER_GID= + +# Shared by every GPU-touching service (llama-server, llama-server-fast, +# comfyui) for group_add: — resolved to real host GIDs by ./scripts/update.sh +# rather than left as plain group names in docker-compose.yml, because Docker +# resolves a *named* group_add entry against the container's own /etc/group, +# not the host's, and fails unpredictably when the image doesn't define one +# (worse with multiple GPU services racing on the same lookup at once — see +# docs/research/rocm-gpu-pin-and-render-group.md and issue #5). Leave blank. +HOST_VIDEO_GID= +HOST_RENDER_GID= # --- llama.cpp / fast model (second, always-resident instance — see # docs/research/fast-model-choice.md and issue #44) --- diff --git a/docker-compose.yml b/docker-compose.yml index 820a9fc..4e18455 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,12 +5,20 @@ services: devices: - /dev/kfd - /dev/dri + # Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in + # .env.example and docs/research/rocm-gpu-pin-and-render-group.md. group_add: - - video - - render + - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}" + - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}" security_opt: - seccomp=unconfined ipc: host + # Caps this process's HIP hardware-queue allocation — works around + # ROCm/ROCm#5706 (GPU pinned at 100%/boost-clock whenever two + # concurrent HIP contexts touch this card, confirmed on real hardware + # against llama-server-fast below). See the research doc above. + environment: + - GPU_MAX_HW_QUEUES=1 volumes: - models:/models command: > @@ -46,11 +54,14 @@ services: - /dev/kfd - /dev/dri group_add: - - video - - render + - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}" + - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}" security_opt: - seccomp=unconfined ipc: host + # See llama-server's identical setting above — same fix, same bug. + environment: + - GPU_MAX_HW_QUEUES=1 volumes: - models:/models command: > @@ -159,23 +170,25 @@ services: devices: - /dev/kfd - /dev/dri + # Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in + # .env.example and docs/research/rocm-gpu-pin-and-render-group.md. group_add: - - video - - render + - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}" + - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}" security_opt: - seccomp=unconfined ipc: host environment: - HSA_OVERRIDE_GFX_VERSION=12.0.1 - PYTORCH_ROCM_ARCH=gfx1201 - # This image manages GPU-group access via GID env vars rather than - # relying solely on group_add above (its own README asks for both) — - # scripts/update.sh resolves these from the host, same pattern as - # SEARXNG_LAN_IP. + # This image also wants GID env vars directly (its own README asks + # for both these and group_add above) — same HOST_VIDEO_GID/ + # HOST_RENDER_GID resolved by scripts/update.sh, shared with + # llama-server/llama-server-fast now instead of comfyui-only vars. - PUID=${COMFYUI_PUID} - PGID=${COMFYUI_PGID} - - VIDEO_GID=${COMFYUI_VIDEO_GID} - - RENDER_GID=${COMFYUI_RENDER_GID} + - VIDEO_GID=${HOST_VIDEO_GID} + - RENDER_GID=${HOST_RENDER_GID} - BASE_STORAGE_PATH=/storage volumes: - comfyui-data:/storage diff --git a/docs/research/rocm-gpu-pin-and-render-group.md b/docs/research/rocm-gpu-pin-and-render-group.md new file mode 100644 index 0000000..c05fe86 --- /dev/null +++ b/docs/research/rocm-gpu-pin-and-render-group.md @@ -0,0 +1,95 @@ +# Research: GPU pinned at 100% with two concurrent llama.cpp containers, and the intermittent "render" group startup error + +**Question:** After adding `llama-server-fast` (#44), real-hardware testing on the R9700 +showed `rocm-smi` pinned at 100% GPU / ~73-101W whenever both `llama-server` and +`llama-server-fast` run concurrently, dropping to 3% / ~25-60W the moment either one +alone is stopped. Separately, `docker compose up` intermittently failed with +`Error response from daemon: unable to find group render: no matching entries in group file` +— confirmed new since the second GPU service was added. See issue #5's comment thread +for the raw `rocm-smi`/`free -h` output this doc is diagnosing. + +## GPU pin: root cause and fix + +**Confirmed via #5's own data**: either container alone is fine (3% GPU, low power). +The pin only appears with two concurrent HIP-context-holding processes on the same +GPU. This matches `ROCm/ROCm#5706` (already flagged as a risk in map #1) — full +comment thread confirms: + +- Root cause: an AMD MES (Micro Engine Scheduler) firmware bug triggered by HIP + hardware-queue creation, pinning the GPU at boost clock the moment ROCm + initializes a queue. Not llama.cpp-specific — reproduced with vLLM and bare + PyTorch ROCm too. Source: [ROCm/ROCm#5706](https://github.com/ROCm/ROCm/issues/5706) + (`tcgu-amd`, AMD engineer, confirms MES firmware root cause; closed as + "fixed" in March, but a report as recent as May 24 shows it recurring even on + patched firmware/kernel). +- **Validated workaround**: `GPU_MAX_HW_QUEUES=1` as a container env var. One + report ran a controlled before/after on the exact image this stack uses + (`ghcr.io/ggml-org/llama.cpp:server-rocm`, R9700/gfx1201): + baseline 100% GPU / 95W → with the var set, 3% GPU / 22W, VRAM unchanged. + Source: same thread, `interconnectedMe`'s comment. +- **Semantics** (why this should apply to our two-container case, not just the + single-process case tested above): `GPU_MAX_HW_QUEUES` is a **per-process** + HIP runtime setting — it caps how many HSA/hardware queues *that process's* + HIP runtime allocates, default higher (over-subscription is what causes the + penalty). Source: [AMD ROCm workload-optimization docs](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html). + Since it's per-process, setting it on *each* container independently is the + correct scope — it should reduce total concurrent hardware-queue creation + across both processes, which is the trigger condition MES chokes on. + **Caveat**: no primary source explicitly tested two concurrent containers + both set to `GPU_MAX_HW_QUEUES=1` — this is a well-grounded extrapolation + from confirmed per-process semantics and the same root-cause mechanism, not + a directly-reproduced fix for our exact topology. Verify with `rocm-smi` + after applying, both containers up. + +## "unable to find group render" — a real Docker bug, not flaky hardware + +This is a known, documented Docker limitation, not something specific to this +stack: `group_add` by **name** requires Docker to resolve that name against +the **container's own** `/etc/group` file — if the image doesn't define a +`render` entry there (common for minimal/slim base images), resolution fails. +Source: [docker/cli#4714](https://github.com/docker/cli/issues/4714) +("`docker run --group-add` by name doesn't add group from host as +documented") and [docker/compose#7277](https://github.com/docker/compose/issues/7277) +(same "no matching entries in group file" error). + +Confirms why it's now intermittent rather than always-broken: this repo's +`docker-compose.yml` uses `group_add: [video, render]` (plain names) on +**three** GPU services now (`llama-server`, `llama-server-fast`, `comfyui`). +Docker Compose starts containers concurrently, and each does its own +name-resolution lookup independently — with only one GPU service before #44, +the resolution almost always won its race; with two (soon three, once +`comfyui`'s downloader/model land per #46) the odds of losing that race and +hitting the unresolved-name path go up. This is consistent with the user's +own observation that it's new since the second GPU service. + +**Fix, already precedented in this repo**: `scripts/update.sh` already +resolves the host's real `video`/`render` **numeric GIDs** for the `comfyui` +service (`COMFYUI_VIDEO_GID`/`COMFYUI_RENDER_GID`, passed as app-level env +vars) — but `comfyui`'s own `group_add:` still uses plain names too, so it +isn't actually protected by that either. The correct fix per the Docker +issues above: use the resolved **numeric GIDs** in `group_add:` itself +(Compose accepts numeric strings directly), not names, on all three GPU +services. Numeric GIDs skip the name-resolution step entirely, eliminating +both the flakiness and the race. + +## Recommendation + +1. Add `GPU_MAX_HW_QUEUES=1` to both `llama-server` and `llama-server-fast`'s + `environment:` blocks. Verify with `rocm-smi` after redeploy, both + containers up — this is the one part of this doc that's extrapolated + rather than directly reproduced, so real confirmation matters here. +2. Resolve host `video`/`render` GIDs once (generalize the existing + `COMFYUI_VIDEO_GID`/`COMFYUI_RENDER_GID` pattern in `scripts/update.sh` + to shared `HOST_VIDEO_GID`/`HOST_RENDER_GID` vars), and switch + `group_add:` on all three GPU services (`llama-server`, + `llama-server-fast`, `comfyui`) from `[video, render]` (names) to the + resolved numeric GIDs. Removes the race entirely rather than reducing its + odds. + +## Sources + +- [ROCm/ROCm#5706 — full comment thread](https://github.com/ROCm/ROCm/issues/5706) +- [AMD ROCm — MI300/MI350 workload optimization docs (GPU_MAX_HW_QUEUES)](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html) +- [docker/cli#4714 — group_add by name doesn't work as documented](https://github.com/docker/cli/issues/4714) +- [docker/compose#7277 — "no matching entries in group file"](https://github.com/docker/compose/issues/7277) +- This repo's issue #5 (real-hardware `rocm-smi`/`free -h` evidence this doc diagnoses) diff --git a/scripts/update.sh b/scripts/update.sh index e465c2b..17dea53 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -56,22 +56,29 @@ else echo "SEARXNG_LAN_IP: couldn't resolve search.home from this host, set it manually if still blank." fi -echo "==> resolving ComfyUI host GID/UID" +echo "==> resolving ComfyUI host UID" # yurisasc/comfyui-rocm7.1 wants these as env vars, not just group_add in # compose — resolve from this host, same pattern as SEARXNG_LAN_IP. set_if_blank COMFYUI_PUID "$(id -u)" set_if_blank COMFYUI_PGID "$(id -g)" + +echo "==> resolving host video/render GIDs (shared by every GPU service)" +# Numeric GIDs, not names, in docker-compose.yml's group_add: — Docker +# resolves a *named* group_add entry against the container's own /etc/group, +# not the host's, and fails unpredictably (worse with multiple GPU services +# starting concurrently and racing on the same lookup) — see +# docs/research/rocm-gpu-pin-and-render-group.md and issue #5. video_gid="$(getent group video 2>/dev/null | cut -d: -f3)" render_gid="$(getent group render 2>/dev/null | cut -d: -f3)" if [ -n "$video_gid" ]; then - set_if_blank COMFYUI_VIDEO_GID "$video_gid" + set_if_blank HOST_VIDEO_GID "$video_gid" else - echo "COMFYUI_VIDEO_GID: no 'video' group on this host, set it manually if still blank." + echo "HOST_VIDEO_GID: no 'video' group on this host, set it manually if still blank." fi if [ -n "$render_gid" ]; then - set_if_blank COMFYUI_RENDER_GID "$render_gid" + set_if_blank HOST_RENDER_GID "$render_gid" else - echo "COMFYUI_RENDER_GID: no 'render' group on this host, set it manually if still blank." + echo "HOST_RENDER_GID: no 'render' group on this host, set it manually if still blank." fi echo "==> git pull"