LLM-Server/docker-compose.yml

services:
  llama-server:
    image: ghcr.io/ggml-org/llama.cpp:server-rocm
    container_name: llama-server
    devices:
      - /dev/kfd
      - /dev/dri
    # Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in
    # .env.example and docs/research/rocm-gpu-pin-and-render-group.md.
    group_add:
      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
    # Caps this process's HIP hardware-queue allocation — works around
    # ROCm/ROCm#5706 (GPU pinned at 100%/boost-clock whenever two
    # concurrent HIP contexts touch this card). See the research doc above.
    environment:
      - GPU_MAX_HW_QUEUES=1
    volumes:
      - models:/models
    command: >
      -m /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
      --host 0.0.0.0
      --port 8080
      --n-gpu-layers ${LLAMA_GPU_LAYERS:-999}
      --ctx-size ${LLAMA_CTX_SIZE:-262144}
      --parallel ${LLAMA_PARALLEL:-2}
      --flash-attn on
      --cache-type-k q8_0
      --cache-type-v q8_0
      --cache-reuse 256
      --jinja
    # --cache-reuse 256: reuse cached KV for any matching prompt chunk of at
    # least 256 tokens (KV-shift, no reprocessing) instead of reprefilling
    # from scratch every request. Directly targets the actual root cause
    # behind the OmniRoute non-ping SSE timeout, not just the symptom — see
    # docs/research/omniroute-non-ping-sse-stream-timeout.md. Pairs with
    # OmniRoute's promptCacheAffinityEnabled (dashboard default), which keeps
    # a conversation's requests pinned to the same slot so there's a matching
    # prefix to reuse.
    # No published host port: llama-server is reached only via the omniroute
    # gateway on the ai-stack docker network now — see issue #15. Its
    # unauthenticated API no longer needs to be LAN-reachable directly.
    expose:
      - "8080"
    restart: unless-stopped
    networks: [ai-stack]
    labels:
      # ponytail: idle-timeout tuning lives here, not in a separate lazytainer config file —
      # one place to look. Raise LAZYTAINER_INACTIVE_TIMEOUT if 15 min proves too eager.
      - "lazytainer.group.llamaserver.sleepMethod=stop"
      - "lazytainer.group.llamaserver.ports=8080"
      - "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
      - "lazytainer.group.llamaserver.minPacketThreshold=2"

  # Dedicated backend for qwen-code's tool-call harmfulness classifier
  # (fastModel in ~/.qwen/settings.json). Was aliased onto llama-server's own
  # 27B connection — every classification call then queued behind whatever
  # heavy generation was already running on that model's 2 GPU slots (issue
  # tracker: OmniRoute semaphore/pr-agent investigation).
  #
  # Tried CPU-only first (own process avoids the GPU queue entirely) — too
  # slow in practice: real classification calls blew past OmniRoute's 60s
  # timeout and retry-looped (504→499→504...). Moved to GPU instead.
  #
  # Context sizing: qwen-code's classifier transcript is hard-capped in its
  # own source (MAX_TRANSCRIPT_MESSAGES=40, MAX_HISTORICAL_ACTION_CHARS=4000
  # per message, packages/core/src/permissions/classifier-transcript.ts) —
  # worst case is ~40-50K tokens, nowhere near the 131072 originally set in
  # settings.json (that number was copied from the main model's entry, not
  # a real qwen-code requirement). 65536 ctx gives ~1.5x margin over that
  # worst case. Qwen3-4B-Instruct-2507 is still the model choice — smallest
  # Qwen3 with long native context (262144) without RoPE-scaling, in case
  # that margin ever needs to grow.
  #
  # VRAM: weights+KV math (~4.9GiB) predicted comfortable headroom in the
  # ~6.1GiB free on the R9700, but measured live it actually used ~5.85GiB —
  # left only ~700MB free, too tight. Dropping --batch-size/--ubatch-size
  # barely moved it (~768MB free) — wrong lever. Actual cause: llama-server
  # runs with --flash-attn on but this service was missing it — without
  # flash attention the unfused attention compute buffer at 65536 ctx is
  # much larger (roughly O(n^2) intermediate buffers vs flash-attn's fused,
  # near-linear workspace), dwarfing the naive weights+KV estimate. Added
  # --flash-attn on to match llama-server; re-verify with rocm-smi after
  # deploy before trusting any of these numbers again. GPU_MAX_HW_QUEUES=1
  # carried over from llama-server's comment above — same ROCm/ROCm#5706
  # clock-pinning bug applies now that two HIP contexts (this +
  # llama-server) share the card.
  #
  # --reasoning off is a no-cost safety net, not a confirmed-needed fix:
  # ggml-org/llama.cpp#20809 (closed) documents some server builds
  # misdetecting Qwen3-Instruct-2507 models as thinking models, routing
  # tool-call output into reasoning_content instead of tool_calls — exactly
  # the failure mode that ruled out the 27B model for this role in the
  # first place. Whether the current image build still has it was never
  # independently confirmed (see docs/research/fast-model-choice.md §4/§6).
  qwen-classifier:
    image: ghcr.io/ggml-org/llama.cpp:server-rocm
    container_name: qwen-classifier
    devices:
      - /dev/kfd
      - /dev/dri
    group_add:
      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
    environment:
      - GPU_MAX_HW_QUEUES=1
    volumes:
      - models:/models
    command: >
      -m /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}
      --host 0.0.0.0
      --port 8080
      --n-gpu-layers 28
      --ctx-size 65536
      --parallel 1
      --batch-size 512
      --ubatch-size 128
      --flash-attn on
      --reasoning off
      --cache-type-k q4_0
      --cache-type-v q4_0
      --jinja
    expose:
      - "8080"
    restart: unless-stopped
    networks: [ai-stack]

  # ponytail: one-off downloader, not a standing service — run via
  # `docker compose --profile tools run --rm downloader`. Folded into
  # scripts/update.sh, which runs this every time; the `test -f` guard is
  # what makes that safe to re-run without re-downloading. Keeps the model
  # files inside the named `models` volume instead of a host bind-mount.
  downloader:
    image: curlimages/curl:latest
    profiles: ["tools"]
    # ponytail: named volume is created root-owned; curl_user (uid 100) can't
    # write into it otherwise, so run as root for this one-off job.
    user: root
    volumes:
      - models:/models
    entrypoint: ["sh", "-c"]
    command:
      - >
        test -f /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} &&
        echo "already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
        https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}

  # Same pattern as downloader above, separate service so this one small
  # file doesn't get re-checked/re-pulled by the big model's job.
  downloader-classifier:
    image: curlimages/curl:latest
    profiles: ["tools"]
    user: root
    volumes:
      - models:/models
    entrypoint: ["sh", "-c"]
    command:
      - >
        test -f /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf} &&
        echo "already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}
        https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}

  # Fetches the three Qwen-Image 2.1 (uncensored GGUF) files ComfyUI needs
  # (diffusion model, text encoder, VAE) — same test -f guard pattern as
  # downloader above. See docs/research/image-generation-model-choice.md and
  # issue #42.
  #
  # GGUF weights go in models/unet (ComfyUI-GGUF's own convention), not
  # models/diffusion_models (that was the old FP8-safetensors layout) — see
  # github.com/city96/ComfyUI-GGUF. Text encoder/VAE stay safetensors, loaded
  # by ComfyUI's normal CLIPLoader/VAELoader, so their folders are unchanged.
  #
  # ponytail: target paths otherwise assume ComfyUI's standard models/ layout
  # under BASE_STORAGE_PATH (/storage) — same "not independently confirmed
  # against the image's Dockerfile" caveat flagged on the comfyui service
  # below. If ComfyUI doesn't pick these up, check its actual models root
  # first.
  downloader-comfyui:
    image: curlimages/curl:latest
    profiles: ["tools"]
    user: root
    volumes:
      - comfyui-data:/storage
    entrypoint: ["sh", "-c"]
    command:
      - >
        mkdir -p /storage/models/unet /storage/models/text_encoders /storage/models/vae &&
        (test -f /storage/models/unet/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen-image-2.1-UC-Q4_K_M.gguf} &&
        echo "diffusion model already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /storage/models/unet/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen-image-2.1-UC-Q4_K_M.gguf}
        https://huggingface.co/abenzerps/Qwen-Image-2.1-Uncensored-GGUF/resolve/main/${COMFYUI_DIFFUSION_MODEL_FILE:-qwen-image-2.1-UC-Q4_K_M.gguf}) &&
        (test -f /storage/models/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen3vl_8b_int8_convrot.safetensors} &&
        echo "text encoder already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /storage/models/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen3vl_8b_int8_convrot.safetensors}
        https://huggingface.co/abenzerps/Qwen-Image-2.1-Uncensored-GGUF/resolve/main/text_encoders/${COMFYUI_TEXT_ENCODER_FILE:-qwen3vl_8b_int8_convrot.safetensors}) &&
        (test -f /storage/models/vae/${COMFYUI_VAE_FILE:-qwen_image_2.1_vae_bf16.safetensors} &&
        echo "vae already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /storage/models/vae/${COMFYUI_VAE_FILE:-qwen_image_2.1_vae_bf16.safetensors}
        https://huggingface.co/abenzerps/Qwen-Image-2.1-Uncensored-GGUF/resolve/main/vae/${COMFYUI_VAE_FILE:-qwen_image_2.1_vae_bf16.safetensors})

  # Local image generation — see issue #38 (wayfinder map). yurisasc's image
  # is gfx1201-tuned specifically (R9700's arch), unlike the official/AMD
  # ComfyUI image which doesn't pin RDNA4 support — see
  # docs/research/image-generation-options.md.
  #
  # Model is Qwen-Image 2.1 GGUF (uncensored variant, see .env.example) —
  # needs the ComfyUI-GGUF custom node (github.com/city96/ComfyUI-GGUF) for
  # its "Unet Loader (GGUF)" node. Not confirmed pre-installed in this image;
  # if missing, exec in and `git clone` it into custom_nodes, then pip
  # install its requirements.txt and restart the container.
  comfyui:
    image: yurisasc/comfyui-rocm7.1:latest
    container_name: comfyui
    devices:
      - /dev/kfd
      - /dev/dri
    # Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in
    # .env.example and docs/research/rocm-gpu-pin-and-render-group.md.
    group_add:
      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
    environment:
      - HSA_OVERRIDE_GFX_VERSION=12.0.1
      - PYTORCH_ROCM_ARCH=gfx1201
    volumes:
      - comfyui-data:/storage
    # yurisasc/comfyui-rocm7.1 has no entrypoint/CMD of its own (just drops
    # into bash) and doesn't read PUID/PGID/VIDEO_GID/RENDER_GID or
    # BASE_STORAGE_PATH — confirmed by inspecting the image directly, so
    # this command is required, not optional, or the container exits 0
    # immediately and restart-loops. Runs as root (image's only user);
    # group_add above is what actually grants /dev/kfd + /dev/dri access.
    #
    # --base-directory fully replaces ComfyUI's custom_nodes/input/output/
    # user/temp dirs with ones under /storage (confirmed in folder_paths.py)
    # rather than adding to the image's baked-in ones — so first boot must
    # seed /storage/custom_nodes from the image's copy (ComfyUI-Manager
    # lives there) or it's gone, and main.py hard-crashes if custom_nodes
    # doesn't exist yet at all.
    entrypoint: ["sh", "-c"]
    command:
      - >
        mkdir -p /storage/custom_nodes /storage/input /storage/output /storage/user &&
        [ -z "$(ls -A /storage/custom_nodes)" ] && cp -r /workspace/ComfyUI/custom_nodes/. /storage/custom_nodes/ || true;
        exec python main.py --listen 0.0.0.0 --port 8188 --base-directory /storage
    #
    # Published host port (unlike llama-server's ai-stack-only pattern):
    # ComfyUI's own UI is meant to be reachable directly too, for a planned
    # external nginx reverse-proxy route to comfy.home — not just through
    # OmniRoute. Still also reachable at http://comfyui:8188 internally on
    # ai-stack, which is the URL to register as OmniRoute's comfyui
    # provider (dashboard or POST /api/providers, per docs/proxy-key-onboarding.md
    # — same undocumented-in-repo manual flow already used for llama-server).
    ports:
      - "8138:8188"
    restart: unless-stopped
    networks: [ai-stack]

  # Replaces litellm — see issue #31 (wayfinder map) for the full migration
  # rationale/findings. No static config.yaml equivalent: provider routing
  # (llama-server, searxng-search) is registered once through the dashboard
  # or POST /api/providers after first boot, not checked into this repo —
  # see docs/proxy-key-onboarding.md.
  omniroute:
    image: diegosouzapw/omniroute:latest
    container_name: omniroute
    depends_on:
      llama-server:
        condition: service_started
    volumes:
      - omniroute-data:/app/data
    env_file: .env
    environment:
      # Split-port mode: dashboard and API are fully separate ports (unlike
      # LiteLLM's single :4000 for both /v1 and /ui) — both published
      # directly below, unlike the old :4000-only host mapping.
      - API_HOST=0.0.0.0
      - API_PORT=${OMNIROUTE_API_PORT:-20129}
      - DASHBOARD_PORT=${OMNIROUTE_DASHBOARD_PORT:-20128}
      # Required to register llama-server/searxng-search as providers —
      # their base URLs are LAN/container-internal addresses, blocked by
      # default (SSRF guard against public-provider spoofing).
      - OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS=true
      - OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS=true
      # Required (production) per docs/reference/ENVIRONMENT.md — shared
      # secret for the internal Codex Responses WebSocket bridge. Missed on
      # first pass; docker-compose config validated fine without it, but
      # the docs are explicit this one's required, not optional.
      - OMNIROUTE_WS_BRIDGE_SECRET=${OMNIROUTE_WS_BRIDGE_SECRET}
      # Default heap (1024MB) is dashboard-only sized per OmniRoute's own
      # Docker guide — every client here is a coding CLI, which needs the
      # larger figure the guide recommends. Paired with mem_limit below.
      - OMNIROUTE_MEMORY_MB=8192
      # Default 300000 (5 min) per OmniRoute's own docs, but this deployment
      # had it dialed down elsewhere (dashboard) to ~95s — too tight for a
      # contended local llama-server: large-context prefill under multiple
      # concurrent slots can outrun that before the first SSE token arrives,
      # so OmniRoute cancels a request that was actually still working (see
      # LLAMA_PARALLEL above for the other half of this fix). Raised here so
      # it's tracked in git instead of a dashboard-only setting.
      - STREAM_IDLE_TIMEOUT_MS=${OMNIROUTE_STREAM_IDLE_TIMEOUT_MS:-180000}
      # Different timer than STREAM_IDLE_TIMEOUT_MS above — that one only
      # bounds gaps *between* SSE chunks once streaming has started.
      # REQUEST_TIMEOUT_MS bounds the wait for the *first* non-ping SSE
      # event, and it's what was still firing ("Stream produced no non-ping
      # SSE event within 95000ms") the morning after the timeout above was
      # raised — see docs/research/omniroute-non-ping-sse-stream-timeout.md.
      # Default 600000 (10 min) per OmniRoute's own docs, but the effective
      # deadline is remaining budget after retries/cooldowns eat into it, not
      # a flat timer, so raised well past the default for headroom.
      - REQUEST_TIMEOUT_MS=${OMNIROUTE_REQUEST_TIMEOUT_MS:-1800000}
    # Same reasoning as litellm's extra_hosts entry below — ai-stack's bridge
    # network can't resolve search.home on its own.
    extra_hosts:
      - "search.home:${SEARXNG_LAN_IP}"
    ports:
      - "${OMNIROUTE_API_PORT:-20129}:${OMNIROUTE_API_PORT:-20129}"
      - "${OMNIROUTE_DASHBOARD_PORT:-20128}:${OMNIROUTE_DASHBOARD_PORT:-20128}"
    # 10+ GiB ceiling per OmniRoute's Docker guide, matching
    # OMNIROUTE_MEMORY_MB=8192 above.
    mem_limit: 10g
    # SQLite WAL needs time to checkpoint back into the main DB file on
    # shutdown — the Docker guide's --stop-timeout 40 equivalent.
    stop_grace_period: 40s
    restart: unless-stopped
    networks: [ai-stack]
    # ponytail: TCP-connect check, not an HTTP /healthz GET — the image has
    # no python3/curl/wget (confirmed live, `which` found only node), and
    # OmniRoute's own Docker guide already treats a bare TCP probe on this
    # port as an acceptable liveness check, not just the HTTP one. Simpler
    # and avoids depending on /healthz's exact path/response shape.
    healthcheck:
      test:
        - CMD-SHELL
        - node -e "require('net').connect(${OMNIROUTE_API_PORT:-20129},'localhost').on('connect',function(){this.end();process.exit(0)}).on('error',()=>process.exit(1))"
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  lazytainer:
    image: ghcr.io/vmorganp/lazytainer:master
    container_name: lazytainer
    # NOT network_mode: host — lazytainer identifies its own container by
    # matching os.Hostname() against the Docker container-ID list
    # (vmorganp/Lazytainer, configureFromLabels()); under host networking the
    # container inherits the host's hostname instead of its own ID, so that
    # match always fails and it panics with "Could not determine container ID
    # of lazytainer" on every start. Host networking also can't see traffic
    # to llama-server:8080 anyway — that port only exists on the ai-stack
    # bridge network (no host port published, see issue #15 above). Joining
    # ai-stack instead fixes both: hostname becomes the real container ID,
    # and it's on the same network as the traffic it's watching.
    networks: [ai-stack]
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
    restart: unless-stopped
    depends_on:
      - llama-server

  # RAG vector store — see docs/agents/... (wayfinder). Dashboard UI published
  # directly like comfyui above, not gatewayed through omniroute (it isn't an
  # LLM provider).
  qdrant:
    image: qdrant/qdrant:latest
    container_name: qdrant
    volumes:
      - qdrant-data:/qdrant/storage
    ports:
      - "6333:6333"
    restart: unless-stopped
    networks: [ai-stack]

  # RAG graph store, native vector index too (can absorb qdrant's job later
  # if the two-DB split proves unnecessary — see wayfinder notes).
  neo4j:
    image: neo4j:5-community
    container_name: neo4j
    environment:
      - NEO4J_AUTH=neo4j/${NEO4J_PASSWORD:?run scripts/update.sh first to resolve this}
    volumes:
      - neo4j-data:/data
    ports:
      - "7474:7474" # browser UI
      - "7687:7687" # bolt
    restart: unless-stopped
    networks: [ai-stack]

  # Chat + image-generation frontend. Talks to ComfyUI directly for images —
  # NOT through omniroute: OmniRoute's built-in `comfyui` provider hardcodes
  # a CheckpointLoaderSimple (single-file-checkpoint) workflow in
  # open-sse/handlers/imageGeneration/providers/comfyUI.ts with no override,
  # so it can't drive Qwen-Image 2.1's split unet/text-encoder/vae GGUF
  # loader chain. Open WebUI's ComfyUI backend is fully generic instead —
  # you upload the exact workflow_api.json exported from ComfyUI's own
  # graph editor and map arbitrary node IDs to parameters.
  #
  # Chat backend still goes through omniroute (OpenAI-compatible), same
  # gateway every other client uses — mint OPENWEBUI_OMNIROUTE_KEY by hand
  # per docs/proxy-key-onboarding.md, same as any other workload.
  #
  # ponytail: only wires the base ComfyUI URL + chat backend via env vars.
  # COMFYUI_WORKFLOW/COMFYUI_WORKFLOW_NODES env vars exist but are reported
  # flaky upstream — set the actual workflow JSON by hand in Settings >
  # Images in the web UI instead, once you've exported it from ComfyUI.
  open-webui:
    image: ghcr.io/open-webui/open-webui:main
    container_name: open-webui
    depends_on:
      comfyui:
        condition: service_started
      omniroute:
        condition: service_started
    environment:
      - OPENAI_API_BASE_URL=http://omniroute:${OMNIROUTE_API_PORT:-20129}/v1
      - OPENAI_API_KEY=${OPENWEBUI_OMNIROUTE_KEY:-}
      - ENABLE_IMAGE_GENERATION=True
      - IMAGE_GENERATION_ENGINE=comfyui
      - COMFYUI_BASE_URL=http://comfyui:8188
      - WEBUI_AUTH=True
    volumes:
      - open-webui-data:/app/backend/data
    ports:
      - "8008:8080"
    restart: unless-stopped
    networks: [ai-stack]

networks:
  ai-stack:

volumes:
  models:
  omniroute-data:
  comfyui-data:
  qdrant-data:
  neo4j-data:
  open-webui-data: