chore: move code comments into docs, leave one-line pointers
Explanations from docker-compose.yml, .env.example and scripts/update.sh now live in docs/architecture.md, gateway.md, network-access.md and operations.md. Code keeps `# see docs/...` pointers, one-line warnings (storage key, SSRF guard, WAL grace period, git pull ordering) and "auto-filled — leave blank" markers. Refs #55 (comment destination map #61) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a0e5e9704b
commit
05effe5c88
4 files changed
+31
-265
No files matched your search
+11
-114
@@ -5,17 +5,13 @@ services:
|
||||
devices:
|
||||
- /dev/kfd
|
||||
- /dev/dri
|
||||
# Numeric GIDs, not names — see HOST_VIDEO_GID/HOST_RENDER_GID in
|
||||
# .env.example and docs/research/rocm-gpu-pin-and-render-group.md.
|
||||
# see docs/architecture.md#gpu-access (group_add GIDs, GPU_MAX_HW_QUEUES)
|
||||
group_add:
|
||||
- "${HOST_VIDEO_GID:?run scripts/update.sh first to resolve this}"
|
||||
- "${HOST_RENDER_GID:?run scripts/update.sh first to resolve this}"
|
||||
security_opt:
|
||||
- seccomp=unconfined
|
||||
ipc: host
|
||||
# Caps this process's HIP hardware-queue allocation — works around
|
||||
# ROCm/ROCm#5706 (GPU pinned at 100%/boost-clock whenever two
|
||||
# concurrent HIP contexts touch this card). See the research doc above.
|
||||
environment:
|
||||
- GPU_MAX_HW_QUEUES=1
|
||||
volumes:
|
||||
@@ -32,63 +28,13 @@ services:
|
||||
--cache-type-v q8_0
|
||||
--cache-reuse 256
|
||||
--jinja
|
||||
# --cache-reuse 256: reuse cached KV for any matching prompt chunk of at
|
||||
# least 256 tokens (KV-shift, no reprocessing) instead of reprefilling
|
||||
# from scratch every request. Directly targets the actual root cause
|
||||
# behind the OmniRoute non-ping SSE timeout, not just the symptom — see
|
||||
# docs/research/omniroute-non-ping-sse-stream-timeout.md. Pairs with
|
||||
# OmniRoute's promptCacheAffinityEnabled (dashboard default), which keeps
|
||||
# a conversation's requests pinned to the same slot so there's a matching
|
||||
# prefix to reuse.
|
||||
# No published host port: llama-server is reached only via the omniroute
|
||||
# gateway on the ai-stack docker network now — see issue #15. Its
|
||||
# unauthenticated API no longer needs to be LAN-reachable directly.
|
||||
# see docs/architecture.md#llama-server (flags), docs/network-access.md#backends (no host port)
|
||||
expose:
|
||||
- "8080"
|
||||
restart: unless-stopped
|
||||
networks: [ai-stack]
|
||||
|
||||
# Dedicated backend for qwen-code's tool-call harmfulness classifier
|
||||
# (fastModel in ~/.qwen/settings.json). Was aliased onto llama-server's own
|
||||
# 27B connection — every classification call then queued behind whatever
|
||||
# heavy generation was already running on that model's 2 GPU slots (issue
|
||||
# tracker: OmniRoute semaphore/pr-agent investigation).
|
||||
#
|
||||
# Tried CPU-only first (own process avoids the GPU queue entirely) — too
|
||||
# slow in practice: real classification calls blew past OmniRoute's 60s
|
||||
# timeout and retry-looped (504→499→504...). Moved to GPU instead.
|
||||
#
|
||||
# Context sizing: qwen-code's classifier transcript is hard-capped in its
|
||||
# own source (MAX_TRANSCRIPT_MESSAGES=40, MAX_HISTORICAL_ACTION_CHARS=4000
|
||||
# per message, packages/core/src/permissions/classifier-transcript.ts) —
|
||||
# worst case is ~40-50K tokens, nowhere near the 131072 originally set in
|
||||
# settings.json (that number was copied from the main model's entry, not
|
||||
# a real qwen-code requirement). 65536 ctx gives ~1.5x margin over that
|
||||
# worst case. Qwen3-4B-Instruct-2507 is still the model choice — smallest
|
||||
# Qwen3 with long native context (262144) without RoPE-scaling, in case
|
||||
# that margin ever needs to grow.
|
||||
#
|
||||
# VRAM: weights+KV math (~4.9GiB) predicted comfortable headroom in the
|
||||
# ~6.1GiB free on the R9700, but measured live it actually used ~5.85GiB —
|
||||
# left only ~700MB free, too tight. Dropping --batch-size/--ubatch-size
|
||||
# barely moved it (~768MB free) — wrong lever. Actual cause: llama-server
|
||||
# runs with --flash-attn on but this service was missing it — without
|
||||
# flash attention the unfused attention compute buffer at 65536 ctx is
|
||||
# much larger (roughly O(n^2) intermediate buffers vs flash-attn's fused,
|
||||
# near-linear workspace), dwarfing the naive weights+KV estimate. Added
|
||||
# --flash-attn on to match llama-server; re-verify with rocm-smi after
|
||||
# deploy before trusting any of these numbers again. GPU_MAX_HW_QUEUES=1
|
||||
# carried over from llama-server's comment above — same ROCm/ROCm#5706
|
||||
# clock-pinning bug applies now that two HIP contexts (this +
|
||||
# llama-server) share the card.
|
||||
#
|
||||
# --reasoning off is a no-cost safety net, not a confirmed-needed fix:
|
||||
# ggml-org/llama.cpp#20809 (closed) documents some server builds
|
||||
# misdetecting Qwen3-Instruct-2507 models as thinking models, routing
|
||||
# tool-call output into reasoning_content instead of tool_calls — exactly
|
||||
# the failure mode that ruled out the 27B model for this role in the
|
||||
# first place. Whether the current image build still has it was never
|
||||
# independently confirmed (see docs/research/fast-model-choice.md §4/§6).
|
||||
# see docs/architecture.md#qwen-classifier
|
||||
qwen-classifier:
|
||||
image: ghcr.io/ggml-org/llama.cpp:server-rocm
|
||||
container_name: qwen-classifier
|
||||
@@ -101,6 +47,7 @@ services:
|
||||
security_opt:
|
||||
- seccomp=unconfined
|
||||
ipc: host
|
||||
# see docs/architecture.md#gpu-access
|
||||
environment:
|
||||
- GPU_MAX_HW_QUEUES=1
|
||||
volumes:
|
||||
@@ -124,16 +71,10 @@ services:
|
||||
restart: unless-stopped
|
||||
networks: [ai-stack]
|
||||
|
||||
# ponytail: one-off downloader, not a standing service — run via
|
||||
# `docker compose --profile tools run --rm downloader`. Folded into
|
||||
# scripts/update.sh, which runs this every time; the `test -f` guard is
|
||||
# what makes that safe to re-run without re-downloading. Keeps the model
|
||||
# files inside the named `models` volume instead of a host bind-mount.
|
||||
# see docs/operations.md#model-downloads (also covers downloader-classifier)
|
||||
downloader:
|
||||
image: curlimages/curl:latest
|
||||
profiles: ["tools"]
|
||||
# ponytail: named volume is created root-owned; curl_user (uid 100) can't
|
||||
# write into it otherwise, so run as root for this one-off job.
|
||||
user: root
|
||||
volumes:
|
||||
- models:/models
|
||||
@@ -145,8 +86,6 @@ services:
|
||||
curl -L --fail --create-dirs -o /models/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
|
||||
https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/${LLAMA_MODEL_FILE:-Qwen3.8-27B-UD-Q4_K_XL.gguf}
|
||||
|
||||
# Same pattern as downloader above, separate service so this one small
|
||||
# file doesn't get re-checked/re-pulled by the big model's job.
|
||||
downloader-classifier:
|
||||
image: curlimages/curl:latest
|
||||
profiles: ["tools"]
|
||||
@@ -161,11 +100,7 @@ services:
|
||||
curl -L --fail --create-dirs -o /models/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}
|
||||
https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/resolve/main/${LLAMA_CLASSIFIER_MODEL_FILE:-Qwen3-4B-Instruct-2507-UD-Q4_K_XL.gguf}
|
||||
|
||||
# Replaces litellm — see issue #31 (wayfinder map) for the full migration
|
||||
# rationale/findings. No static config.yaml equivalent: provider routing
|
||||
# (llama-server, searxng-search) is registered once through the dashboard
|
||||
# or POST /api/providers after first boot, not checked into this repo —
|
||||
# see docs/proxy-key-onboarding.md.
|
||||
# see docs/architecture.md#omniroute; runtime config lives in its dashboard, see docs/gateway.md
|
||||
omniroute:
|
||||
image: diegosouzapw/omniroute:latest
|
||||
container_name: omniroute
|
||||
@@ -176,64 +111,28 @@ services:
|
||||
- omniroute-data:/app/data
|
||||
env_file: .env
|
||||
environment:
|
||||
# Split-port mode: dashboard and API are fully separate ports (unlike
|
||||
# LiteLLM's single :4000 for both /v1 and /ui) — both published
|
||||
# directly below, unlike the old :4000-only host mapping.
|
||||
- API_HOST=0.0.0.0
|
||||
- API_PORT=${OMNIROUTE_API_PORT:-20129}
|
||||
- DASHBOARD_PORT=${OMNIROUTE_DASHBOARD_PORT:-20128}
|
||||
# Required to register llama-server/searxng-search as providers —
|
||||
# their base URLs are LAN/container-internal addresses, blocked by
|
||||
# default (SSRF guard against public-provider spoofing).
|
||||
# Security: relaxes OmniRoute's SSRF guard — see docs/gateway.md#provider-connections
|
||||
- OMNIROUTE_ALLOW_PRIVATE_PROVIDER_URLS=true
|
||||
- OMNIROUTE_ALLOW_LOCAL_PROVIDER_URLS=true
|
||||
# Required (production) per docs/reference/ENVIRONMENT.md — shared
|
||||
# secret for the internal Codex Responses WebSocket bridge. Missed on
|
||||
# first pass; docker-compose config validated fine without it, but
|
||||
# the docs are explicit this one's required, not optional.
|
||||
- OMNIROUTE_WS_BRIDGE_SECRET=${OMNIROUTE_WS_BRIDGE_SECRET}
|
||||
# Default heap (1024MB) is dashboard-only sized per OmniRoute's own
|
||||
# Docker guide — every client here is a coding CLI, which needs the
|
||||
# larger figure the guide recommends. Paired with mem_limit below.
|
||||
- OMNIROUTE_MEMORY_MB=8192
|
||||
# Default 300000 (5 min) per OmniRoute's own docs, but this deployment
|
||||
# had it dialed down elsewhere (dashboard) to ~95s — too tight for a
|
||||
# contended local llama-server: large-context prefill under multiple
|
||||
# concurrent slots can outrun that before the first SSE token arrives,
|
||||
# so OmniRoute cancels a request that was actually still working (see
|
||||
# LLAMA_PARALLEL above for the other half of this fix). Raised here so
|
||||
# it's tracked in git instead of a dashboard-only setting.
|
||||
# see docs/gateway.md#timeouts (both timeouts below)
|
||||
- STREAM_IDLE_TIMEOUT_MS=${OMNIROUTE_STREAM_IDLE_TIMEOUT_MS:-180000}
|
||||
# Different timer than STREAM_IDLE_TIMEOUT_MS above — that one only
|
||||
# bounds gaps *between* SSE chunks once streaming has started.
|
||||
# REQUEST_TIMEOUT_MS bounds the wait for the *first* non-ping SSE
|
||||
# event, and it's what was still firing ("Stream produced no non-ping
|
||||
# SSE event within 95000ms") the morning after the timeout above was
|
||||
# raised — see docs/research/omniroute-non-ping-sse-stream-timeout.md.
|
||||
# Default 600000 (10 min) per OmniRoute's own docs, but the effective
|
||||
# deadline is remaining budget after retries/cooldowns eat into it, not
|
||||
# a flat timer, so raised well past the default for headroom.
|
||||
- REQUEST_TIMEOUT_MS=${OMNIROUTE_REQUEST_TIMEOUT_MS:-1800000}
|
||||
# Same reasoning as litellm's extra_hosts entry below — ai-stack's bridge
|
||||
# network can't resolve search.home on its own.
|
||||
# see docs/gateway.md#web-search
|
||||
extra_hosts:
|
||||
- "search.home:${SEARXNG_LAN_IP}"
|
||||
ports:
|
||||
- "${OMNIROUTE_API_PORT:-20129}:${OMNIROUTE_API_PORT:-20129}"
|
||||
- "${OMNIROUTE_DASHBOARD_PORT:-20128}:${OMNIROUTE_DASHBOARD_PORT:-20128}"
|
||||
# 10+ GiB ceiling per OmniRoute's Docker guide, matching
|
||||
# OMNIROUTE_MEMORY_MB=8192 above.
|
||||
mem_limit: 10g
|
||||
# SQLite WAL needs time to checkpoint back into the main DB file on
|
||||
# shutdown — the Docker guide's --stop-timeout 40 equivalent.
|
||||
# Don't shorten: SQLite needs it to checkpoint its WAL, or recent writes are lost — see docs/architecture.md#omniroute
|
||||
stop_grace_period: 40s
|
||||
restart: unless-stopped
|
||||
networks: [ai-stack]
|
||||
# ponytail: TCP-connect check, not an HTTP /healthz GET — the image has
|
||||
# no python3/curl/wget (confirmed live, `which` found only node), and
|
||||
# OmniRoute's own Docker guide already treats a bare TCP probe on this
|
||||
# port as an acceptable liveness check, not just the HTTP one. Simpler
|
||||
# and avoids depending on /healthz's exact path/response shape.
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
@@ -243,9 +142,7 @@ services:
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
|
||||
# RAG vector store — see docs/agents/... (wayfinder). Dashboard UI published
|
||||
# directly, not gatewayed through omniroute (it isn't an
|
||||
# LLM provider).
|
||||
# see docs/architecture.md#qdrant, docs/network-access.md#qdrant
|
||||
qdrant:
|
||||
image: qdrant/qdrant:latest
|
||||
container_name: qdrant
|
||||
|
||||
Reference in new issue
Block a user