LLM-Server/docker-compose.yml

services:
  llama-server:
    image: ghcr.io/ggml-org/llama.cpp:server-rocm
    container_name: llama-server
    devices:
      - /dev/kfd
      - /dev/dri
    # see docs/architecture.md#gpu-access (group_add GIDs, GPU_MAX_HW_QUEUES)
    group_add:
      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
    environment:
      - GPU_MAX_HW_QUEUES=1
    volumes:
      - models:/models
    command: >
      -m /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
      --host 0.0.0.0
      --port 8080
      --n-gpu-layers ${LLAMA_GPU_LAYERS:-999}
      --ctx-size ${LLAMA_CTX_SIZE:-262144}
      --parallel ${LLAMA_PARALLEL:-2}
      --kv-unified
      --flash-attn on
      --cache-type-k q8_0
      --cache-type-v q8_0
      --cache-reuse 256
      --spec-type draft-mtp 
      --spec-draft-n-max 2
      --jinja
    # see docs/architecture.md#llama-server (flags), docs/network-access.md#backends (no host port)
    expose:
      - "8080"
    restart: unless-stopped
    networks: [ai-stack]

  # see docs/architecture.md#qwen-classifier
  qwen-classifier:
    image: ghcr.io/ggml-org/llama.cpp:server-rocm
    container_name: qwen-classifier
    devices:
      - /dev/kfd
      - /dev/dri
    group_add:
      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
    # see docs/architecture.md#gpu-access
    environment:
      - GPU_MAX_HW_QUEUES=1
    volumes:
      - models:/models
    command: >
      -m /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}
      --host 0.0.0.0
      --port 8080
      --n-gpu-layers 28
      --ctx-size 50000
      --parallel 1
      --batch-size 512
      --ubatch-size 128
      --flash-attn on
      --reasoning off
      --cache-type-k q4_0
      --cache-type-v q4_0
      --jinja
    expose:
      - "8080"
    restart: unless-stopped
    networks: [ai-stack]

  # see docs/operations.md#model-downloads (also covers downloader-classifier)
  downloader:
    image: curlimages/curl:latest
    profiles: ["tools"]
    user: root
    volumes:
      - models:/models
    entrypoint: ["sh", "-c"]
    command:
      - >
        test -f /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf} &&
        echo "already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
        https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}

  downloader-classifier:
    image: curlimages/curl:latest
    profiles: ["tools"]
    user: root
    volumes:
      - models:/models
    entrypoint: ["sh", "-c"]
    command:
      - >
        test -f /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf} &&
        echo "already downloaded, skipping" ||
        curl -L --fail --create-dirs -o /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}
        https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}

  # see docs/architecture.md#omniroute; runtime config lives in its dashboard, see docs/gateway.md
  omniroute:
    image: diegosouzapw/omniroute:latest
    container_name: omniroute
    depends_on:
      llama-server:
        condition: service_started
    volumes:
      - omniroute-data:/app/data
    env_file: .env
    environment:
      - API_HOST=0.0.0.0
      - API_PORT=${OMNIROUTE_API_PORT:-20129}
      - DASHBOARD_PORT=${OMNIROUTE_DASHBOARD_PORT:-20128}
      # Security: relaxes OmniRoute's SSRF guard — see docs/gateway.md#provider-connections
      - OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS=true
      - OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS=true
      - OMNIROUTE_WS_BRIDGE_SECRET=${OMNIROUTE_WS_BRIDGE_SECRET}
      - OMNIROUTE_MEMORY_MB=8192
      # see docs/gateway.md#timeouts (both timeouts below)
      - STREAM_IDLE_TIMEOUT_MS=${OMNIROUTE_STREAM_IDLE_TIMEOUT_MS:-180000}
      - REQUEST_TIMEOUT_MS=${OMNIROUTE_REQUEST_TIMEOUT_MS:-1800000}
    # see docs/gateway.md#web-search
    extra_hosts:
      - "search.home:${SEARXNG_LAN_IP}"
    ports:
      - "${OMNIROUTE_API_PORT:-20129}:${OMNIROUTE_API_PORT:-20129}"
      - "${OMNIROUTE_DASHBOARD_PORT:-20128}:${OMNIROUTE_DASHBOARD_PORT:-20128}"
    mem_limit: 10g
    # Don't shorten: SQLite needs it to checkpoint its WAL, or recent writes are lost — see docs/architecture.md#omniroute
    stop_grace_period: 40s
    restart: unless-stopped
    networks: [ai-stack]
    healthcheck:
      test:
        - CMD-SHELL
        - node -e "require('net').connect(${OMNIROUTE_API_PORT:-20129},'localhost').on('connect',function(){this.end();process.exit(0)}).on('error',()=>process.exit(1))"
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  # see docs/architecture.md#qdrant, docs/network-access.md#qdrant
  qdrant:
    image: qdrant/qdrant:latest
    container_name: qdrant
    volumes:
      - qdrant-data:/qdrant/storage
    ports:
      - "127.0.0.1:6333:6333"
    restart: unless-stopped
    networks: [ai-stack]

networks:
  ai-stack:

volumes:
  models:
  omniroute-data:
  qdrant-data: