LLM-Server/scripts/update.sh

#!/usr/bin/env bash
# The one command to run after any change to this repo (compose file, .env,
# or a git pull) to bring the running stack in sync. Ensures secrets/keys
# exist, pulls, validates, rebuilds/re-pulls images, and recreates only what
# changed — safe to run any time, including with nothing to do.
#
# Tunable config values (LLAMA_*, ports, timeouts — anything with a real
# default in .env.example) are synced from .env.example every run. A value
# already matching is left alone silently. A value that DIFFERS from the
# server's current .env is a conflict: interactively, you're shown every
# conflict on one screen (via gum) and choose which to accept — unpicked
# keys keep the server's current value. Non-interactively (no TTY — cron,
# CI, piped), any conflict is a hard error unless --force is passed, which
# accepts every new value automatically. Secrets and host-resolved values
# (blank in .env.example — OMNIROUTE_*_SECRET/_KEY/_SALT/_PASSWORD,
# SEARXNG_LAN_IP, COMFYUI_PUID/PGID, HOST_VIDEO_GID/RENDER_GID) are never
# touched by this — they keep going through set_if_blank as before.
#
# omniroute's own routing/provider config (llama-server, search) lives in
# its dashboard, not a checked-in file like the old litellm-config.yaml —
# see issue #31 and docs/proxy-key-onboarding.md.
#
# ponytail: no rollback/backup logic — this is a single-user homelab box,
# not a fleet. If a bad config lands, `git revert` + re-run is the recovery
# path, not this script.
set -euo pipefail
cd "$(dirname "$0")/.."

FORCE=false
for arg in "$@"; do
  case "$arg" in
    --force) FORCE=true ;;
    *) echo "Usage: $0 [--force]" >&2; exit 1 ;;
  esac
done

# Must run before anything else touches a file this script itself reads
# (docker-compose.yml, .env.example, this script's own remaining lines) —
# a self-updating script isn't guaranteed atomic against its own file
# changing mid-run, so pulling later can execute a mix of old and new
# script/compose content in one pass. Bit us for real: old GID-resolution
# code ran, then this pulled in new var names docker-compose.yml now
# requires, and nothing re-ran the (now-current) resolution step for
# them — see issue #5's thread.
echo "==> git pull"
git pull --ff-only

[ -f .env ] || cp .env.example .env

echo "==> syncing tracked config values from .env.example"
# ponytail: gum (charmbracelet/gum) is a single static binary. Vendored as
# a release tarball under scripts/vendor/ (checked into git) for the R9700
# box, which has no outbound internet access — the download fallback below
# is only for other archs / when the vendored copy is missing or stale.
# Cached under .cache/gum/ (gitignored) so repeat runs don't re-extract.
GUM_VERSION="0.14.5"
GUM_DIR="$(pwd)/.cache/gum"
GUM_BIN="$GUM_DIR/gum"
ensure_gum() {
  command -v gum >/dev/null 2>&1 && { echo "gum"; return; }
  [ -x "$GUM_BIN" ] && { echo "$GUM_BIN"; return; }
  mkdir -p "$GUM_DIR"
  local arch tmpdir vendored
  case "$(uname -m)" in
    x86_64) arch="x86_64" ;;
    aarch64|arm64) arch="arm64" ;;
    *) echo "no gum build for $(uname -m), falling back to plain prompts" >&2; echo ""; return ;;
  esac
  tmpdir="$(mktemp -d)"
  vendored="$(pwd)/scripts/vendor/gum_${GUM_VERSION}_Linux_${arch}.tar.gz"
  if [ -f "$vendored" ]; then
    tar -xz -C "$tmpdir" -f "$vendored"
  else
    local url="https://github.com/charmbracelet/gum/releases/download/v${GUM_VERSION}/gum_${GUM_VERSION}_Linux_${arch}.tar.gz"
    if ! curl -fsSL "$url" | tar -xz -C "$tmpdir" 2>/dev/null; then
      echo "no vendored gum for $arch and couldn't download from $url (no internet egress? falling back to plain prompts)" >&2
    fi
  fi
  if [ -n "$(find "$tmpdir" -name gum -type f 2>/dev/null)" ]; then
    find "$tmpdir" -name gum -type f -exec cp {} "$GUM_BIN" \;
    chmod +x "$GUM_BIN" 2>/dev/null || true
  fi
  rm -rf "$tmpdir"
  [ -x "$GUM_BIN" ] && echo "$GUM_BIN" || echo ""
}

# Collect every key where .env.example has a real (non-blank) default:
# missing from .env -> just add it (no conflict, nothing to decide);
# present and identical -> leave alone silently; present and different ->
# a conflict to resolve below.
conflict_keys=()
conflict_old=()
conflict_new=()
while IFS='=' read -r key value; do
  [ -n "$value" ] || continue
  current="$(grep -E "^${key}=" .env | head -1 | cut -d= -f2-)"
  if ! grep -qE "^${key}=" .env; then
    echo "${key}=${value}" >> .env
  elif [ "$current" != "$value" ]; then
    conflict_keys+=("$key")
    conflict_old+=("$current")
    conflict_new+=("$value")
  fi
done < <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=.+' .env.example)

if [ "${#conflict_keys[@]}" -gt 0 ]; then
  if [ "$FORCE" = true ]; then
    for i in "${!conflict_keys[@]}"; do
      key="${conflict_keys[$i]}"; new="${conflict_new[$i]}"
      sed -i "s|^${key}=.*|${key}=${new}|" .env
      echo "${key}: ${conflict_old[$i]} -> ${new} (--force)"
    done
  elif [ ! -t 0 ] || [ ! -t 1 ]; then
    echo "ERROR: ${#conflict_keys[@]} config value(s) in .env differ from .env.example, and this isn't an interactive terminal:" >&2
    for i in "${!conflict_keys[@]}"; do
      echo "  ${conflict_keys[$i]}: ${conflict_old[$i]} (current) vs ${conflict_new[$i]} (.env.example)" >&2
    done
    echo "Re-run interactively to choose per-key, or pass --force to accept every new value." >&2
    exit 1
  else
    gum_bin="$(ensure_gum)"
    labels=()
    for i in "${!conflict_keys[@]}"; do
      labels+=("${conflict_keys[$i]}: ${conflict_old[$i]} -> ${conflict_new[$i]}")
    done
    if [ -n "$gum_bin" ]; then
      selected="$(printf '%s\n' "${labels[@]}" | "$gum_bin" choose --no-limit --selected "$(printf '%s\n' "${labels[@]}" | paste -sd,)" --header "Config differs from .env.example — selected keys take the new value, unselected keep the server's current value:")"
    else
      # ponytail: plain-bash fallback if gum couldn't be fetched (offline,
      # unsupported arch) — same one-screen-of-conflicts idea, cruder UI.
      echo "Config differs from .env.example. Enter space-separated numbers to KEEP the server's current value (all others take the new value), or press enter to take every new value:"
      for i in "${!conflict_keys[@]}"; do
        echo "  $((i+1))) ${labels[$i]}"
      done
      read -r -p "> " keep_nums
      selected=""
      for i in "${!conflict_keys[@]}"; do
        case " $keep_nums " in
          *" $((i+1)) "*) ;;
          *) selected="${selected}${labels[$i]}"$'\n' ;;
        esac
      done
    fi
    for i in "${!conflict_keys[@]}"; do
      key="${conflict_keys[$i]}"; new="${conflict_new[$i]}"
      if printf '%s\n' "$selected" | grep -qxF "${labels[$i]}"; then
        sed -i "s|^${key}=.*|${key}=${new}|" .env
        echo "${key}: ${conflict_old[$i]} -> ${new}"
      else
        echo "${key}: kept ${conflict_old[$i]} (server value)"
      fi
    done
  fi
fi

# Handles all three cases: the KEY=value line is missing entirely (.env
# predates that var being added to .env.example — sed can't fix what isn't
# there, so this appends it), present but blank, or already set.
set_if_blank() {
  local key="$1" value="$2"
  if grep -qE "^${key}=.*[^[:space:]]" .env; then
    echo "${key}: already set, skipping."
  elif grep -qE "^${key}=" .env; then
    sed -i "s|^${key}=.*|${key}=${value}|" .env
    echo "${key}: set."
  else
    echo "${key}=${value}" >> .env
    echo "${key}: added (was missing from .env)."
  fi
}

echo "==> filling in missing secrets"
# Random values — safe to re-run, never overwrites what's already set.
# OMNIROUTE_STORAGE_ENCRYPTION_KEY especially: never change it after first
# run, existing encrypted data becomes unreadable if you do (same caveat as
# LiteLLM's old LITELLM_SALT_KEY).
set_if_blank OMNIROUTE_INITIAL_PASSWORD "$(openssl rand -hex 16)"
set_if_blank OMNIROUTE_JWT_SECRET "$(openssl rand -base64 48)"
set_if_blank OMNIROUTE_API_KEY_SECRET "$(openssl rand -hex 32)"
set_if_blank OMNIROUTE_STORAGE_ENCRYPTION_KEY "$(openssl rand -hex 32)"
set_if_blank OMNIROUTE_MACHINE_ID_SALT "$(openssl rand -hex 16)"
set_if_blank OMNIROUTE_CLI_SALT "$(openssl rand -hex 16)"
set_if_blank OMNIROUTE_WS_BRIDGE_SECRET "$(openssl rand -hex 32)"
set_if_blank NEO4J_PASSWORD "$(openssl rand -hex 16)"

echo "==> resolving SEARXNG_LAN_IP"
# search.home is a LAN mDNS/local-DNS name — resolvable from this host, just
# not from inside the omniroute container (see docs/research/litellm-searxng-search.md,
# still the relevant background even though omniroute replaced litellm — see issue #31).
searxng_ip="$(getent hosts search.home 2>/dev/null | awk '{print $1}' | head -1)"
if [ -n "$searxng_ip" ]; then
  set_if_blank SEARXNG_LAN_IP "$searxng_ip"
else
  echo "SEARXNG_LAN_IP: couldn't resolve search.home from this host, set it manually if still blank."
fi

echo "==> resolving ComfyUI host UID"
# yurisasc/comfyui-rocm7.1 wants these as env vars, not just group_add in
# compose — resolve from this host, same pattern as SEARXNG_LAN_IP.
set_if_blank COMFYUI_PUID "$(id -u)"
set_if_blank COMFYUI_PGID "$(id -g)"

echo "==> resolving host video/render GIDs (shared by every GPU service)"
# Numeric GIDs, not names, in docker-compose.yml's group_add: — Docker
# resolves a *named* group_add entry against the container's own /etc/group,
# not the host's, and fails unpredictably (worse with multiple GPU services
# starting concurrently and racing on the same lookup) — see
# docs/research/rocm-gpu-pin-and-render-group.md and issue #5.
video_gid="$(getent group video 2>/dev/null | cut -d: -f3)"
render_gid="$(getent group render 2>/dev/null | cut -d: -f3)"
if [ -n "$video_gid" ]; then
  set_if_blank HOST_VIDEO_GID "$video_gid"
else
  echo "HOST_VIDEO_GID: no 'video' group on this host, set it manually if still blank."
fi
if [ -n "$render_gid" ]; then
  set_if_blank HOST_RENDER_GID "$render_gid"
else
  echo "HOST_RENDER_GID: no 'render' group on this host, set it manually if still blank."
fi

echo "==> validating compose config"
docker compose config -q

echo "==> pulling images"
docker compose pull --ignore-buildable

echo "==> rebuilding local-build services"
docker compose build --pull

echo "==> ensuring models are downloaded (skips already-present files)"
docker compose --profile tools run --rm downloader
docker compose --profile tools run --rm downloader-fast
docker compose --profile tools run --rm downloader-comfyui

echo "==> bringing up omniroute"
docker compose up -d --wait omniroute

# ponytail: no scripted key-minting yet, unlike the old LiteLLM /key/generate
# flow — omniroute's POST /api/keys needs a dashboard login session
# (ManagementSessionAuth), not a static bearer key, and that flow hasn't
# been verified against a live instance (see issue #37). No in-stack
# workload needs a key right now (nothing left calls the gateway besides
# coding CLIs, which mint their own by hand per docs/proxy-key-onboarding.md)
# — revisit this script once that flow is automatable.

echo "==> recreating changed services"
docker compose up -d --remove-orphans

echo "==> status"
docker compose ps