diff --git a/.env.example b/.env.example index 0d36a98..014c0d1 100644 --- a/.env.example +++ b/.env.example @@ -113,3 +113,12 @@ LLAMA_FAST_GPU_LAYERS=999 # 8192 keeps this instance's KV cache negligible. LLAMA_FAST_CTX_SIZE=8192 LLAMA_FAST_PARALLEL=2 + +# --- ComfyUI diffusion model (Qwen-Image, FP8 — see docs/research/ +# image-generation-model-choice.md and issue #42) --- +# Three files: diffusion weights, text encoder, VAE — all from the official +# Comfy-Org FP8 split, chosen specifically because it's the only candidate +# with a ComfyUI workflow pre-validated on this exact GPU (gfx1201/R9700). +COMFYUI_DIFFUSION_MODEL_FILE=qwen_image_fp8_e4m3fn.safetensors +COMFYUI_TEXT_ENCODER_FILE=qwen_2.5_vl_7b_fp8_scaled.safetensors +COMFYUI_VAE_FILE=qwen_image_vae.safetensors diff --git a/docker-compose.yml b/docker-compose.yml index 1bcda03..820a9fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,6 +116,39 @@ services: curl -L --fail --create-dirs -o /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} + # Fetches the three Qwen-Image FP8 files ComfyUI needs (diffusion model, + # text encoder, VAE) — same test -f guard pattern as downloader/ + # downloader-fast above. See docs/research/image-generation-model-choice.md + # and issue #42. + # + # ponytail: target paths assume ComfyUI's standard models/ layout under + # BASE_STORAGE_PATH (/storage) — same "not independently confirmed against + # the image's Dockerfile" caveat already flagged on the comfyui service + # below. If ComfyUI doesn't pick these up, check its actual models root + # first. + downloader-comfyui: + image: curlimages/curl:latest + profiles: ["tools"] + user: root + volumes: + - comfyui-data:/storage + entrypoint: ["sh", "-c"] + command: + - > + mkdir -p /storage/models/diffusion_models /storage/models/text_encoders /storage/models/vae && + (test -f /storage/models/diffusion_models/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen_image_fp8_e4m3fn.safetensors} && + echo "diffusion model already downloaded, skipping" || + curl -L --fail --create-dirs -o /storage/models/diffusion_models/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen_image_fp8_e4m3fn.safetensors} + https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/diffusion_models/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen_image_fp8_e4m3fn.safetensors}) && + (test -f /storage/models/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen_2.5_vl_7b_fp8_scaled.safetensors} && + echo "text encoder already downloaded, skipping" || + curl -L --fail --create-dirs -o /storage/models/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen_2.5_vl_7b_fp8_scaled.safetensors} + https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen_2.5_vl_7b_fp8_scaled.safetensors}) && + (test -f /storage/models/vae/${COMFYUI_VAE_FILE:-qwen_image_vae.safetensors} && + echo "vae already downloaded, skipping" || + curl -L --fail --create-dirs -o /storage/models/vae/${COMFYUI_VAE_FILE:-qwen_image_vae.safetensors} + https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/vae/${COMFYUI_VAE_FILE:-qwen_image_vae.safetensors}) + # Local image generation — see issue #38 (wayfinder map). yurisasc's image # is gfx1201-tuned specifically (R9700's arch), unlike the official/AMD # ComfyUI image which doesn't pin RDNA4 support — see diff --git a/scripts/switch-model.sh b/scripts/switch-model.sh new file mode 100644 index 0000000..8aec6b6 --- /dev/null +++ b/scripts/switch-model.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Swap GPU residency between llama-server (Qwen) and comfyui — they never +# run concurrently, VRAM doesn't fit both (see issue #38's map). Manual +# invocation only, no auto-switching. +# +# Bypasses lazytainer entirely and drives docker compose directly — its +# idle-stop can't be used for this. Root cause (see +# docs/research/lazytainer-omniroute-idle-stop.md, issue #40): lazytainer's +# packet-threshold detector is source-blind and can't tell OmniRoute's +# periodic health-check pings apart from real traffic on the same port, so +# it never reliably sleeps a service on its own. A scripted swap always +# knows which service should go up/down, so it doesn't need that heuristic. +# +# llama-server-fast (the small classifier model, issue #44) is NOT part of +# this swap — it's meant to stay always-resident. Worst case with comfyui up +# is comfyui (~25GB, Qwen-Image FP8) + llama-server-fast (~5GB) ≈ 30GB, +# still under the 32GB card but tight — unverified on real hardware, check +# `docker compose ps` / VRAM usage after the first real swap. +# +# OmniRoute may show the just-stopped provider as errored/offline in its +# dashboard for up to CREDENTIAL_HEALTH_CHECK_INTERVAL (default 5 min) after +# a swap — cosmetic, not a functional problem (see the research doc above). +set -euo pipefail +cd "$(dirname "$0")/.." + +usage() { + echo "Usage: $0 {qwen|comfyui}" >&2 + echo " qwen - stop comfyui, start llama-server" >&2 + echo " comfyui - stop llama-server, start comfyui" >&2 + exit 1 +} + +[ $# -eq 1 ] || usage + +case "$1" in + qwen) + from=comfyui + to=llama-server + ;; + comfyui) + from=llama-server + to=comfyui + ;; + *) + usage + ;; +esac + +echo "==> stopping $from" +docker compose stop "$from" + +echo "==> starting $to" +docker compose up -d "$to" + +echo "==> status" +docker compose ps diff --git a/scripts/update.sh b/scripts/update.sh index 2ddd34f..e465c2b 100755 --- a/scripts/update.sh +++ b/scripts/update.sh @@ -89,6 +89,7 @@ docker compose build --pull echo "==> ensuring models are downloaded (skips already-present files)" docker compose --profile tools run --rm downloader docker compose --profile tools run --rm downloader-fast +docker compose --profile tools run --rm downloader-comfyui echo "==> bringing up omniroute" docker compose up -d --wait omniroute