LLM-Server/docker-compose.yml

services:
  llama-server:
    image: ghcr.io/ggml-org/llama.cpp:server-rocm
    container_name: llama-server
    devices:
      - /dev/kfd
      - /dev/dri
    group_add:
      - video
      - render
    security_opt:
      - seccomp=unconfined
    ipc: host
    volumes:
      - models:/models
    command: >
      -m /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
      --host 0.0.0.0
      --port 8080
      --n-gpu-layers ${LLAMA_GPU_LAYERS:-999}
      --ctx-size ${LLAMA_CTX_SIZE:-131072}
      --jinja
    # No published host port: llama-server is reached only via the omniroute
    # gateway on the ai-stack docker network now — see issue #15. Its
    # unauthenticated API no longer needs to be LAN-reachable directly.
    expose:
      - "8080"
    restart: unless-stopped
    networks: [ai-stack]
    labels:
      # ponytail: idle-timeout tuning lives here, not in a separate lazytainer config file —
      # one place to look. Raise LAZYTAINER_INACTIVE_TIMEOUT if 15 min proves too eager.
      - "lazytainer.group.llamaserver.sleepMethod=stop"
      - "lazytainer.group.llamaserver.ports=8080"
      - "lazytainer.group.llamaserver.inactiveTimeout=${LAZYTAINER_INACTIVE_TIMEOUT:-900}"
      - "lazytainer.group.llamaserver.minPacketThreshold=2"

  # ponytail: one-off downloader, not a standing service — run via
  # `docker compose --profile tools run --rm downloader`. Folded into
  # scripts/update.sh, which runs this every time; the `test -f` guard is
  # what makes that safe to re-run without re-downloading. Keeps the model
  # files inside the named `models` volume instead of a host bind-mount.
  downloader:
    image: curlimages/curl:latest
    profiles: ["tools"]
    # ponytail: named volume is created root-owned; curl_user (uid 100) can't
    # write into it otherwise, so run as root for this one-off job.
    user: root
    volumes:
      - models:/models
    entrypoint: ["sh", "-c"]
    command:
      - >
        test -f /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} &&
        echo "already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
        https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf};
        test -f /models/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf} &&
        echo "already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /models/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf}
        https://huggingface.co/ggml-org/bge-small-en-v1.5-Q8_0-GGUF/resolve/main/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf}

  # Dedicated embedding model for OmniRoute's memory feature. llama.cpp loads
  # one model per process and the already-running Qwen3.8-27B chat model
  # isn't embedding-trained, so this is a second, separate llama-server
  # instance rather than adding --embeddings to the chat one — see
  # docs/research/litellm-knowledgebase.md #3. bge-small-en-v1.5 outputs
  # 384-dim vectors — use that as the Qdrant collection size.
  # ponytail: CPU image, no GPU devices — a 33M-param embedding model is
  # fast enough on CPU and this avoids VRAM contention with llama-server's
  # 27B chat model on the same GPU.
  embedding-server:
    image: ghcr.io/ggml-org/llama.cpp:server
    container_name: embedding-server
    volumes:
      - models:/models
    command: >
      -m /models/${EMBEDDING_MODEL_FILE:-bge-small-en-v1.5-q8_0.gguf}
      --host 0.0.0.0
      --port 8081
      --embeddings
      --pooling mean
    restart: unless-stopped
    networks: [ai-stack]

  # Vector store for OmniRoute's memory feature — wired up in the OmniRoute
  # dashboard as a memory service, not via static config here. Not published
  # to the host: only OmniRoute (same ai-stack network) talks to it.
  qdrant:
    image: qdrant/qdrant:latest
    container_name: qdrant
    volumes:
      - qdrant-data:/qdrant/storage
    restart: unless-stopped
    networks: [ai-stack]
    healthcheck:
      test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/localhost/6333'"]
      interval: 10s
      timeout: 5s
      retries: 5

  # Replaces litellm — see issue #31 (wayfinder map) for the full migration
  # rationale/findings. No static config.yaml equivalent: provider routing
  # (llama-server, searxng-search) is registered once through the dashboard
  # or POST /api/providers after first boot, not checked into this repo —
  # see docs/proxy-key-onboarding.md.
  omniroute:
    image: diegosouzapw/omniroute:latest
    container_name: omniroute
    depends_on:
      llama-server:
        condition: service_started
      qdrant:
        condition: service_healthy
    volumes:
      - omniroute-data:/app/data
    env_file: .env
    environment:
      # Split-port mode: dashboard and API are fully separate ports (unlike
      # LiteLLM's single :4000 for both /v1 and /ui) — both published
      # directly below, unlike the old :4000-only host mapping.
      - API_HOST=0.0.0.0
      - API_PORT=${OMNIROUTE_API_PORT:-20129}
      - DASHBOARD_PORT=${OMNIROUTE_DASHBOARD_PORT:-20128}
      # Required to register llama-server/searxng-search as providers —
      # their base URLs are LAN/container-internal addresses, blocked by
      # default (SSRF guard against public-provider spoofing).
      - OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS=true
      - OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS=true
      # Required (production) per docs/reference/ENVIRONMENT.md — shared
      # secret for the internal Codex Responses WebSocket bridge. Missed on
      # first pass; docker-compose config validated fine without it, but
      # the docs are explicit this one's required, not optional.
      - OMNIROUTE_WS_BRIDGE_SECRET=${OMNIROUTE_WS_BRIDGE_SECRET}
      # Default heap (1024MB) is dashboard-only sized per OmniRoute's own
      # Docker guide — every client here is a coding CLI, which needs the
      # larger figure the guide recommends. Paired with mem_limit below.
      - OMNIROUTE_MEMORY_MB=8192
    # Same reasoning as litellm's extra_hosts entry below — ai-stack's bridge
    # network can't resolve search.home on its own.
    extra_hosts:
      - "search.home:${SEARXNG_LAN_IP}"
    ports:
      - "${OMNIROUTE_API_PORT:-20129}:${OMNIROUTE_API_PORT:-20129}"
      - "${OMNIROUTE_DASHBOARD_PORT:-20128}:${OMNIROUTE_DASHBOARD_PORT:-20128}"
    # 10+ GiB ceiling per OmniRoute's Docker guide, matching
    # OMNIROUTE_MEMORY_MB=8192 above.
    mem_limit: 10g
    # SQLite WAL needs time to checkpoint back into the main DB file on
    # shutdown — the Docker guide's --stop-timeout 40 equivalent.
    stop_grace_period: 40s
    restart: unless-stopped
    networks: [ai-stack]
    # ponytail: TCP-connect check, not an HTTP /healthz GET — the image has
    # no python3/curl/wget (confirmed live, `which` found only node), and
    # OmniRoute's own Docker guide already treats a bare TCP probe on this
    # port as an acceptable liveness check, not just the HTTP one. Simpler
    # and avoids depending on /healthz's exact path/response shape.
    healthcheck:
      test:
        - CMD-SHELL
        - node -e "require('net').connect(${OMNIROUTE_API_PORT:-20129},'localhost').on('connect',function(){this.end();process.exit(0)}).on('error',()=>process.exit(1))"
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  lazytainer:
    image: ghcr.io/vmorganp/lazytainer:master
    container_name: lazytainer
    # NOT network_mode: host — lazytainer identifies its own container by
    # matching os.Hostname() against the Docker container-ID list
    # (vmorganp/Lazytainer, configureFromLabels()); under host networking the
    # container inherits the host's hostname instead of its own ID, so that
    # match always fails and it panics with "Could not determine container ID
    # of lazytainer" on every start. Host networking also can't see traffic
    # to llama-server:8080 anyway — that port only exists on the ai-stack
    # bridge network (no host port published, see issue #15 above). Joining
    # ai-stack instead fixes both: hostname becomes the real container ID,
    # and it's on the same network as the traffic it's watching.
    networks: [ai-stack]
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
    restart: unless-stopped
    depends_on:
      - llama-server

networks:
  ai-stack:

volumes:
  models:
  omniroute-data:
  qdrant-data: