LLM-Server/docker-compose.yml

services:
  llama-server:
    image: ghcr.io/ggml-org/llama.cpp:server-rocm
    container_name: llama-server
    devices:
      - /dev/kfd
      - /dev/dri
    # Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in
    # .env.example and docs/research/rocm-gpu-pin-and-render-group.md.
    group_add:
      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
    # Caps this process's HIP hardware-queue allocation — works around
    # ROCm/ROCm#5706 (GPU pinned at 100%/boost-clock whenever two
    # concurrent HIP contexts touch this card, confirmed on real hardware
    # against llama-server-fast below). See the research doc above.
    environment:
      - GPU_MAX_HW_QUEUES=1
    volumes:
      - models:/models
    command: >
      -m /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
      --host 0.0.0.0
      --port 8080
      --n-gpu-layers ${LLAMA_GPU_LAYERS:-999}
      --ctx-size ${LLAMA_CTX_SIZE:-262144}
      --parallel ${LLAMA_PARALLEL:-2}
      --flash-attn on
      --cache-type-k q8_0
      --cache-type-v q8_0
      --jinja
    # No published host port: llama-server is reached only via the omniroute
    # gateway on the ai-stack docker network now — see issue #15. Its
    # unauthenticated API no longer needs to be LAN-reachable directly.
    expose:
      - "8080"
    restart: unless-stopped
    networks: [ai-stack]
    labels:
      # ponytail: idle-timeout tuning lives here, not in a separate lazytainer config file —
      # one place to look. Raise LAZYTAINER_INACTIVE_TIMEOUT if 15 min proves too eager.
      - "lazytainer.group.llamaserver.sleepMethod=stop"
      - "lazytainer.group.llamaserver.ports=8080"
      - "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
      - "lazytainer.group.llamaserver.minPacketThreshold=2"

  llama-server-fast:
    image: ghcr.io/ggml-org/llama.cpp:server-rocm
    container_name: llama-server-fast
    devices:
      - /dev/kfd
      - /dev/dri
    group_add:
      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
    # See llama-server's identical setting above — same fix, same bug.
    environment:
      - GPU_MAX_HW_QUEUES=1
    volumes:
      - models:/models
    command: >
      -m /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
      --host 0.0.0.0
      --port 8080
      --n-gpu-layers ${LLAMA_FAST_GPU_LAYERS:-999}
      --ctx-size ${LLAMA_FAST_CTX_SIZE:-8192}
      --parallel ${LLAMA_FAST_PARALLEL:-1}
      --flash-attn on
      --cache-type-k q8_0
      --cache-type-v q8_0
      --reasoning off
      --jinja
    # Second, always-resident llama.cpp instance — small non-thinking model
    # used as qwen-code's Auto Mode classifier fastModel, alongside the main
    # 27B instance above. See docs/research/fast-model-choice.md and #44.
    # Same ai-stack-only pattern as llama-server: no published host port.
    expose:
      - "8080"
    restart: unless-stopped
    networks: [ai-stack]
    labels:
      - "lazytainer.group.llamaserverfast.sleepMethod=stop"
      - "lazytainer.group.llamaserverfast.ports=8080"
      - "lazytainer.group.llamaserverfast.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
      - "lazytainer.group.llamaserverfast.minPacketThreshold=2"

  # ponytail: one-off downloader, not a standing service — run via
  # `docker compose --profile tools run --rm downloader`. Folded into
  # scripts/update.sh, which runs this every time; the `test -f` guard is
  # what makes that safe to re-run without re-downloading. Keeps the model
  # files inside the named `models` volume instead of a host bind-mount.
  downloader:
    image: curlimages/curl:latest
    profiles: ["tools"]
    # ponytail: named volume is created root-owned; curl_user (uid 100) can't
    # write into it otherwise, so run as root for this one-off job.
    user: root
    volumes:
      - models:/models
    entrypoint: ["sh", "-c"]
    command:
      - >
        test -f /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} &&
        echo "already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
        https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}

  # Same test -f guard pattern as downloader above — fetches the second,
  # smaller model for llama-server-fast. See issue #44.
  downloader-fast:
    image: curlimages/curl:latest
    profiles: ["tools"]
    user: root
    volumes:
      - models:/models
    entrypoint: ["sh", "-c"]
    command:
      - >
        test -f /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf} &&
        echo "already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /models/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}
        https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_FAST_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q8_K_XL.gguf}

  # Fetches the three Qwen-Image FP8 files ComfyUI needs (diffusion model,
  # text encoder, VAE) — same test -f guard pattern as downloader/
  # downloader-fast above. See docs/research/image-generation-model-choice.md
  # and issue #42.
  #
  # ponytail: target paths assume ComfyUI's standard models/ layout under
  # BASE_STORAGE_PATH (/storage) — same "not independently confirmed against
  # the image's Dockerfile" caveat already flagged on the comfyui service
  # below. If ComfyUI doesn't pick these up, check its actual models root
  # first.
  downloader-comfyui:
    image: curlimages/curl:latest
    profiles: ["tools"]
    user: root
    volumes:
      - comfyui-data:/storage
    entrypoint: ["sh", "-c"]
    command:
      - >
        mkdir -p /storage/models/diffusion_models /storage/models/text_encoders /storage/models/vae &&
        (test -f /storage/models/diffusion_models/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen_image_fp8_e4m3fn.safetensors} &&
        echo "diffusion model already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /storage/models/diffusion_models/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen_image_fp8_e4m3fn.safetensors}
        https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/diffusion_models/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen_image_fp8_e4m3fn.safetensors}) &&
        (test -f /storage/models/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen_2.5_vl_7b_fp8_scaled.safetensors} &&
        echo "text encoder already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /storage/models/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen_2.5_vl_7b_fp8_scaled.safetensors}
        https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen_2.5_vl_7b_fp8_scaled.safetensors}) &&
        (test -f /storage/models/vae/${COMFYUI_VAE_FILE:-qwen_image_vae.safetensors} &&
        echo "vae already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /storage/models/vae/${COMFYUI_VAE_FILE:-qwen_image_vae.safetensors}
        https://huggingface.co/Comfy-Org/Qwen-Image_ComfyUI/resolve/main/split_files/vae/${COMFYUI_VAE_FILE:-qwen_image_vae.safetensors})

  # Local image generation — see issue #38 (wayfinder map). yurisasc's image
  # is gfx1201-tuned specifically (R9700's arch), unlike the official/AMD
  # ComfyUI image which doesn't pin RDNA4 support — see
  # docs/research/image-generation-options.md.
  comfyui:
    image: yurisasc/comfyui-rocm7.1:latest
    container_name: comfyui
    devices:
      - /dev/kfd
      - /dev/dri
    # Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in
    # .env.example and docs/research/rocm-gpu-pin-and-render-group.md.
    group_add:
      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
    environment:
      - HSA_OVERRIDE_GFX_VERSION=12.0.1
      - PYTORCH_ROCM_ARCH=gfx1201
      # This image also wants GID env vars directly (its own README asks
      # for both these and group_add above) — same HOST_VIDEO_GID/
      # HOST_RENDER_GID resolved by scripts/update.sh, shared with
      # llama-server/llama-server-fast now instead of comfyui-only vars.
      - PUID=${COMFYUI_PUID}
      - PGID=${COMFYUI_PGID}
      - VIDEO_GID=${HOST_VIDEO_GID}
      - RENDER_GID=${HOST_RENDER_GID}
      - BASE_STORAGE_PATH=/storage
    volumes:
      - comfyui-data:/storage
    # ponytail: exact internal storage path taken from the image's own
    # BASE_STORAGE_PATH env var, not independently confirmed against its
    # Dockerfile — if models/workflows don't persist across a recreate,
    # check this against the image's actual entrypoint first.
    #
    # Published host port (unlike llama-server's ai-stack-only pattern):
    # ComfyUI's own UI is meant to be reachable directly too, for a planned
    # external nginx reverse-proxy route to comfy.home — not just through
    # OmniRoute. Still also reachable at http://comfyui:8188 internally on
    # ai-stack, which is the URL to register as OmniRoute's comfyui
    # provider (dashboard or POST /api/providers, per docs/proxy-key-onboarding.md
    # — same undocumented-in-repo manual flow already used for llama-server).
    ports:
      - "8138:8188"
    restart: unless-stopped
    networks: [ai-stack]

  # Replaces litellm — see issue #31 (wayfinder map) for the full migration
  # rationale/findings. No static config.yaml equivalent: provider routing
  # (llama-server, searxng-search) is registered once through the dashboard
  # or POST /api/providers after first boot, not checked into this repo —
  # see docs/proxy-key-onboarding.md.
  omniroute:
    image: diegosouzapw/omniroute:latest
    container_name: omniroute
    depends_on:
      llama-server:
        condition: service_started
      llama-server-fast:
        condition: service_started
    volumes:
      - omniroute-data:/app/data
    env_file: .env
    environment:
      # Split-port mode: dashboard and API are fully separate ports (unlike
      # LiteLLM's single :4000 for both /v1 and /ui) — both published
      # directly below, unlike the old :4000-only host mapping.
      - API_HOST=0.0.0.0
      - API_PORT=${OMNIROUTE_API_PORT:-20129}
      - DASHBOARD_PORT=${OMNIROUTE_DASHBOARD_PORT:-20128}
      # Required to register llama-server/searxng-search as providers —
      # their base URLs are LAN/container-internal addresses, blocked by
      # default (SSRF guard against public-provider spoofing).
      - OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS=true
      - OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS=true
      # Required (production) per docs/reference/ENVIRONMENT.md — shared
      # secret for the internal Codex Responses WebSocket bridge. Missed on
      # first pass; docker-compose config validated fine without it, but
      # the docs are explicit this one's required, not optional.
      - OMNIROUTE_WS_BRIDGE_SECRET=${OMNIROUTE_WS_BRIDGE_SECRET}
      # Default heap (1024MB) is dashboard-only sized per OmniRoute's own
      # Docker guide — every client here is a coding CLI, which needs the
      # larger figure the guide recommends. Paired with mem_limit below.
      - OMNIROUTE_MEMORY_MB=8192
      # Default 300000 (5 min) per OmniRoute's own docs, but this deployment
      # had it dialed down elsewhere (dashboard) to ~95s — too tight for a
      # contended local llama-server: large-context prefill under multiple
      # concurrent slots can outrun that before the first SSE token arrives,
      # so OmniRoute cancels a request that was actually still working (see
      # LLAMA_PARALLEL above for the other half of this fix). Raised here so
      # it's tracked in git instead of a dashboard-only setting.
      - STREAM_IDLE_TIMEOUT_MS=${OMNIROUTE_STREAM_IDLE_TIMEOUT_MS:-180000}
    # Same reasoning as litellm's extra_hosts entry below — ai-stack's bridge
    # network can't resolve search.home on its own.
    extra_hosts:
      - "search.home:${SEARXNG_LAN_IP}"
    ports:
      - "${OMNIROUTE_API_PORT:-20129}:${OMNIROUTE_API_PORT:-20129}"
      - "${OMNIROUTE_DASHBOARD_PORT:-20128}:${OMNIROUTE_DASHBOARD_PORT:-20128}"
    # 10+ GiB ceiling per OmniRoute's Docker guide, matching
    # OMNIROUTE_MEMORY_MB=8192 above.
    mem_limit: 10g
    # SQLite WAL needs time to checkpoint back into the main DB file on
    # shutdown — the Docker guide's --stop-timeout 40 equivalent.
    stop_grace_period: 40s
    restart: unless-stopped
    networks: [ai-stack]
    # ponytail: TCP-connect check, not an HTTP /healthz GET — the image has
    # no python3/curl/wget (confirmed live, `which` found only node), and
    # OmniRoute's own Docker guide already treats a bare TCP probe on this
    # port as an acceptable liveness check, not just the HTTP one. Simpler
    # and avoids depending on /healthz's exact path/response shape.
    healthcheck:
      test:
        - CMD-SHELL
        - node -e "require('net').connect(${OMNIROUTE_API_PORT:-20129},'localhost').on('connect',function(){this.end();process.exit(0)}).on('error',()=>process.exit(1))"
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  lazytainer:
    image: ghcr.io/vmorganp/lazytainer:master
    container_name: lazytainer
    # NOT network_mode: host — lazytainer identifies its own container by
    # matching os.Hostname() against the Docker container-ID list
    # (vmorganp/Lazytainer, configureFromLabels()); under host networking the
    # container inherits the host's hostname instead of its own ID, so that
    # match always fails and it panics with "Could not determine container ID
    # of lazytainer" on every start. Host networking also can't see traffic
    # to llama-server:8080 anyway — that port only exists on the ai-stack
    # bridge network (no host port published, see issue #15 above). Joining
    # ai-stack instead fixes both: hostname becomes the real container ID,
    # and it's on the same network as the traffic it's watching.
    networks: [ai-stack]
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
    restart: unless-stopped
    depends_on:
      - llama-server

networks:
  ai-stack:

volumes:
  models:
  omniroute-data:
  comfyui-data: