LLM-Server/docker-compose.yml

services:
  llama-server:
    image: ghcr.io/ggml-org/llama.cpp:server-rocm
    container_name: llama-server
    devices:
      - /dev/kfd
      - /dev/dri
    # see docs/architecture.md#gpu-access (group_add GIDs, GPU_MAX_HW_QUEUES)
    group_add:
      - "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
      - "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
    security_opt:
      - seccomp=unconfined
    ipc: host
    environment:
      - GPU_MAX_HW_QUEUES=1
    volumes:
      - models:/models
    command: >
      -m /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
      --host 0.0.0.0
      --port 8080
      --n-gpu-layers ${LLAMA_GPU_LAYERS:-999}
      --ctx-size ${LLAMA_CTX_SIZE:-262144}
      --parallel ${LLAMA_PARALLEL:-2}
      --kv-unified
      --flash-attn on
      --cache-type-k q8_0
      --cache-type-v q8_0
      --cache-ram ${LLAMA_CACHE_RAM:-24576}
      --spec-type draft-dflash
      -md /models/${LLAMA_DRAFT_MODEL_FILE:-Qwen3.8-27B-DFlash2-Q8_0.gguf}
      --spec-draft-n-max ${LLAMA_SPEC_N_MAX:-7}
      --jinja
    # see docs/architecture.md#llama-server (flags), docs/network-access.md#backends (no host port)
    expose:
      - "8080"
    restart: unless-stopped
    networks: [ai-stack]

  # see docs/operations.md#model-downloads; the list of files is models.yml
  downloader:
    image: alpine:3
    profiles: ["tools"]
    user: root
    volumes:
      - models:/models
      - ./models.yml:/models.yml:ro
    entrypoint: ["sh", "-euo", "pipefail", "-c"]
    command:
      - |
        apk add --no-cache -q curl yq-go
        yq -r '.models[] | (.file // "") + " " + (.url // "")' /models.yml | while read -r file url; do
          if [ -z "$$url" ]; then echo "models.yml: entry '$$file' needs both file and url" >&2; exit 1; fi
          if [ -f "/models/$$file" ]; then echo "$$file: already downloaded, skipping"; continue; fi
          curl -L --fail -o "/models/$$file.part" "$$url"
          mv "/models/$$file.part" "/models/$$file"
        done

  # see docs/operations.md#speed-bench
  speed-bench:
    profiles: ["tools"]
    build: ./speed-bench
    entrypoint: ["python", "/app/speed_bench.py", "--url", "http://llama-server:8080"]
    volumes:
      - hf-cache:/root/.cache/huggingface
      - ./bench-results:/results
    networks: [ai-stack]

  # see docs/architecture.md#omniroute; runtime config lives in its dashboard, see docs/gateway.md
  omniroute:
    image: diegosouzapw/omniroute:latest
    container_name: omniroute
    depends_on:
      llama-server:
        condition: service_started
    volumes:
      - omniroute-data:/app/data
    env_file: .env
    environment:
      - API_HOST=0.0.0.0
      - API_PORT=${OMNIROUTE_API_PORT:-20129}
      - DASHBOARD_PORT=${OMNIROUTE_DASHBOARD_PORT:-20128}
      # Security: relaxes OmniRoute's SSRF guard — see docs/gateway.md#provider-connections
      - OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS=true
      - OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS=true
      - OMNIROUTE_WS_BRIDGE_SECRET=${OMNIROUTE_WS_BRIDGE_SECRET}
      - OMNIROUTE_MEMORY_MB=8192
      # see docs/gateway.md#timeouts (both timeouts below)
      - STREAM_IDLE_TIMEOUT_MS=${OMNIROUTE_STREAM_IDLE_TIMEOUT_MS:-180000}
      - REQUEST_TIMEOUT_MS=${OMNIROUTE_REQUEST_TIMEOUT_MS:-1800000}
    # see docs/gateway.md#web-search
    extra_hosts:
      - "search.home:${SEARXNG_LAN_IP}"
    ports:
      - "${OMNIROUTE_API_PORT:-20129}:${OMNIROUTE_API_PORT:-20129}"
      - "${OMNIROUTE_DASHBOARD_PORT:-20128}:${OMNIROUTE_DASHBOARD_PORT:-20128}"
    mem_limit: 10g
    # Don't shorten: SQLite needs it to checkpoint its WAL, or recent writes are lost — see docs/architecture.md#omniroute
    stop_grace_period: 40s
    restart: unless-stopped
    networks: [ai-stack]
    healthcheck:
      test:
        - CMD-SHELL
        - node -e "require('net').connect(${OMNIROUTE_API_PORT:-20129},'localhost').on('connect',function(){this.end();process.exit(0)}).on('error',()=>process.exit(1))"
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  # see docs/architecture.md#qdrant, docs/network-access.md#qdrant
  qdrant:
    image: qdrant/qdrant:latest
    container_name: qdrant
    volumes:
      - qdrant-data:/qdrant/storage
    ports:
      - "127.0.0.1:6333:6333"
    restart: unless-stopped
    networks: [ai-stack]

networks:
  ai-stack:

volumes:
  models:
  omniroute-data:
  qdrant-data:
  hf-cache: