Merge pull request 'fix(publish,docker): fix Blender runtime, CI disk/retry, local build docs' (#1) from feat/auto-godot-blender-tracking into main

2026-08-06 20:04:05 +00:00
parent 1d15d6950f 5564948d0a
commit 2071119c0d
8 changed files with 416 additions and 19 deletions
@@ -1,30 +1,116 @@
 name: Publish godot-ci image

 on:
-  push:
-    branches: [main]
-    paths: [Dockerfile]
+  schedule:
+    # Midnight daily. Picks up new Godot stable releases (and, incidentally, new
+    # Blender stables whenever it rebuilds — see resolve step below).
+    - cron: '0 0 * * *'
  workflow_dispatch:
+    inputs:
+      godot_version:
+        description: >-
+          Godot version to (re)build, e.g. 4.7.1 — must exist as a
+          barichello/godot-ci tag (https://hub.docker.com/r/barichello/godot-ci/tags).
+          Leave empty to build the newest stable, same as the nightly run.
+        required: false
+        type: string

 jobs:
  publish:
    runs-on: ubuntu-latest
+    env:
+      IMAGE: git.arthurerlich.de/haylan/godot-ci
+      DOCKER_CLI_EXPERIMENTAL: enabled
    steps:
      - uses: actions/checkout@v4

      - name: Log in to Gitea registry
        run: echo "${{ secrets.REGISTRY_PUSH_PAT }}" | docker login git.arthurerlich.de -u haylan --password-stdin

-      - name: Build and push
+      - name: Resolve target Godot version
+        id: resolve
        run: |
-          docker build -t git.arthurerlich.de/haylan/godot-ci:4.7.1 .
-          docker push git.arthurerlich.de/haylan/godot-ci:4.7.1
+          set -euo pipefail
+
+          if [ -n "${{ inputs.godot_version }}" ]; then
+            VERSION="${{ inputs.godot_version }}"
+            UPDATE_LATEST=false
+          else
+            VERSION="$(./scripts/resolve-godot-version.sh)"
+            UPDATE_LATEST=true
+          fi
+
+          if ! docker manifest inspect "barichello/godot-ci:${VERSION}" >/dev/null 2>&1; then
+            if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+              echo "::error::No barichello/godot-ci tag for '${VERSION}'. Check https://hub.docker.com/r/barichello/godot-ci/tags"
+              exit 1
+            fi
+            echo "::notice::Newest Godot stable is ${VERSION} but barichello/godot-ci has no matching tag yet — skipping, will retry tomorrow."
+            echo "skip=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          if [ "$UPDATE_LATEST" = true ] && docker manifest inspect "${IMAGE}:${VERSION}" >/dev/null 2>&1; then
+            echo "::notice::${VERSION} is already published and nothing changed — skipping."
+            echo "skip=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+          echo "update_latest=${UPDATE_LATEST}" >> "$GITHUB_OUTPUT"
+          echo "skip=false" >> "$GITHUB_OUTPUT"
+
+      - name: Resolve newest Blender
+        if: steps.resolve.outputs.skip != 'true'
+        id: blender
+        run: echo "url=$(./scripts/resolve-blender-url.sh)" >> "$GITHUB_OUTPUT"
+
+      # Now that this workflow tracks the newest Godot/Blender instead of one
+      # pinned version, most nightly runs build genuinely new multi-GB layers
+      # instead of hitting cache — nothing here ever pruned old ones, so the
+      # runner's disk fills up over successive runs until a push (a blob
+      # write) is what finally fails. Clear stale images/build cache first.
+      - name: Free up runner disk
+        if: steps.resolve.outputs.skip != 'true'
+        run: docker system prune -af --volumes
+
+      - name: Build and push
+        if: steps.resolve.outputs.skip != 'true'
+        run: |
+          set -euo pipefail
+          VERSION="${{ steps.resolve.outputs.version }}"
+          docker build \
+            --build-arg GODOT_VERSION="$VERSION" \
+            --build-arg BLENDER_URL="${{ steps.blender.outputs.url }}" \
+            -t "${IMAGE}:${VERSION}" .
+
+          # Registry blob PUTs have been failing with transient 500s
+          # (unrelated to build correctness) — retry a few times before
+          # giving up.
+          push_with_retry() {
+            for attempt in 1 2 3; do
+              if docker push "$1"; then
+                return 0
+              fi
+              echo "::warning::docker push $1 failed (attempt $attempt/3), retrying..."
+              sleep $((attempt * 10))
+            done
+            return 1
+          }
+
+          push_with_retry "${IMAGE}:${VERSION}"
+
+          if [ "${{ steps.resolve.outputs.update_latest }}" = "true" ]; then
+            docker tag "${IMAGE}:${VERSION}" "${IMAGE}:latest"
+            push_with_retry "${IMAGE}:latest"
+          fi

      # Gitea packages belong to the owner, not a repo, by default — pushing
      # the image alone does NOT make it show up under this repo's Packages
      # tab. Link it explicitly. Non-fatal: a failure here (e.g. already
      # linked) shouldn't fail a build whose push already succeeded.
      - name: Link package to this repository
+        if: steps.resolve.outputs.skip != 'true'
        run: |
          curl -sS -X POST \
            -H "Authorization: token ${{ secrets.REGISTRY_PUSH_PAT }}" \
@@ -1,4 +1,12 @@
-FROM barichello/godot-ci:4.7.1
+# GODOT_VERSION selects the barichello/godot-ci base tag (Godot 4.0+ only — see
+# docs/handoff/godot-ci-custom-image.md). BLENDER_URL is the tarball to bake in;
+# .gitea/workflows/publish.yml always resolves both at build time via
+# scripts/resolve-godot-version.sh and scripts/resolve-blender-url.sh. The
+# defaults below are only a fallback for a plain local `docker build`.
+ARG GODOT_VERSION=4.7.1
+FROM barichello/godot-ci:${GODOT_VERSION}
+
+ARG BLENDER_URL=https://download.blender.org/release/Blender4.2/blender-4.2.3-linux-x64.tar.xz

 RUN apt-get update && apt-get install -y --no-install-recommends \
    libfontconfig1 \
@@ -6,7 +14,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
    libx11-dev libxcursor-dev libxinerama-dev libgl1-mesa-dev \
    libasound2-dev libpulse-dev libudev-dev libxi-dev libxrandr-dev libwayland-dev \
    mingw-w64 \
-    blender \
+    # Blender's own runtime deps, not pulled in by anything above — without
+    # these `blender --version` fails with "libxkbcommon.so.0: cannot open
+    # shared object file" (caught via a local `docker build`, see README).
+    libxkbcommon0 libsm6 libice6 \
  && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix \
  && update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix \
-  && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/* \
+  # Blender is not bundled in barichello/godot-ci, and Ubuntu's apt package lags
+  # (stuck on 4.0.2 on noble) — install the resolved tarball straight from
+  # download.blender.org instead of `apt-get install blender`.
+  && curl -fsSL "$BLENDER_URL" -o /tmp/blender.tar.xz \
+  && mkdir -p /opt/blender \
+  && tar -xJf /tmp/blender.tar.xz -C /opt/blender --strip-components=1 \
+  && ln -sf /opt/blender/blender /usr/local/bin/blender \
+  && rm /tmp/blender.tar.xz
@@ -1,20 +1,38 @@
 # godot-ci

-Custom image extending `barichello/godot-ci:4.7.1` with the Godot 4.7.1 export-template toolchain
-pre-installed, so `Project4x`'s `export-template.yml` doesn't `apt-get install` it on every release
-run.
+Custom image extending `barichello/godot-ci` with the Godot export-template toolchain and Blender
+pre-installed, so `Project4x`'s `export-template.yml` doesn't `apt-get install` them on every
+release run.

-Built from the handoff spec at `Project4x`'s `docs/handoff/godot-ci-custom-image.md`.
+Built from the handoff spec at `Project4x`'s `docs/handoff/godot-ci-custom-image.md`; scope and
+versioning have since moved on from that spec — see below.

 ## Image

-`git.arthurerlich.de/haylan/godot-ci:4.7.1`
+`git.arthurerlich.de/haylan/godot-ci:<godot-version>`, e.g. `git.arthurerlich.de/haylan/godot-ci:4.7.1`,
+plus a floating `git.arthurerlich.de/haylan/godot-ci:latest` that always points at the newest.

-Published automatically by `.gitea/workflows/publish.yml` on every push to `main` that touches
-`Dockerfile`, or manually via workflow dispatch.
+Godot **4.0+** only (Blender's `.blend` import is a Godot 4 feature — 3.x builds have no use for
+this image's Blender toolchain).

-A Godot version bump means updating `Dockerfile`'s `FROM` line and the tag in both the workflow
-and this README together.
+### How versions get built
+
+`.gitea/workflows/publish.yml`:
+
+- **Nightly (midnight, `schedule:`)**: resolves the newest *stable* Godot release
+  (`scripts/resolve-godot-version.sh`) and the newest stable Blender
+  (`scripts/resolve-blender-url.sh`), and rebuilds+pushes `<version>` + `latest` only if that
+  Godot version isn't already published. If `barichello/godot-ci` hasn't tagged the new Godot
+  version yet, the run skips (with a notice, not a failure) and retries the next night.
+- **Manual (`workflow_dispatch`)**: type a Godot version (e.g. `4.6.3`) to (re)build and push that
+  exact tag, always with whatever Blender is newest at build time. Overwrites the tag if it
+  already exists. Does **not** touch `latest`. Fails clearly if `barichello/godot-ci` has no
+  matching tag — check
+  [its tag list](https://hub.docker.com/r/barichello/godot-ci/tags) first.
+
+Blender has no official "latest stable" API — `resolve-blender-url.sh` scrapes
+`download.blender.org/release/`. `scripts/test-lib.sh` is an offline smoke test for the parsing
+logic both resolve scripts share (`scripts/lib.sh`).

 ## Auth

@@ -27,9 +45,29 @@ cannot push to the package registry on this instance.
 - **Owner:** TODO — fill in once the token is generated (Settings → Applications → Generate New
  Token on the account that should own the published packages).

+## Building locally for testing
+
+The `Dockerfile`'s `GODOT_VERSION` and `BLENDER_URL` build args both have defaults, so a plain
+build with no args works for local testing:
+
+```sh
+docker build -t godot-ci-local .
+```
+
+Override either to test a specific combination, matching what `publish.yml` would resolve for a
+given night:
+
+```sh
+docker build \
+  --build-arg GODOT_VERSION=4.6.3 \
+  --build-arg BLENDER_URL=https://download.blender.org/release/Blender4.2/blender-4.2.3-linux-x64.tar.xz \
+  -t godot-ci-local .
+```
+
 ## Consuming this image

-In `Project4x`'s `.gitea/workflows/export-template.yml`:
+In `Project4x`'s `.gitea/workflows/export-template.yml`, pin to a specific version (recommended,
+matches whatever `GODOT_VERSION` the project itself targets) or float on `latest`:

 ```diff
 - container: barichello/godot-ci:4.7.1
@@ -0,0 +1,173 @@
+# Handoff spec: custom godot-ci image
+
+Status: **done, and superseded** — the `godot-ci` repo this spec asked for exists and is built.
+Its versioning has since moved past what's written below: instead of one Dockerfile pinned to
+`4.7.1`, it now auto-tracks the newest stable Godot (4.0+) and newest stable Blender on a nightly
+schedule, with a manual `workflow_dispatch` to (re)build any specific version. See that repo's
+`README.md` for the current behavior; this file is kept for the original problem statement and
+scope rationale, not as a source of truth on tags/versions. Decided via wayfinder map
+[#158](https://git.arthurerlich.de/haylan/Project4x/issues/158), ticket
+[#162](https://git.arthurerlich.de/haylan/Project4x/issues/162).
+
+## Problem
+
+`export-template.yml`'s `export` job runs in `barichello/godot-ci:4.7.1` and, on every single run,
+`apt-get install`s a from-source Godot export-template toolchain before it can build anything. That
+install is pure dead time repeated on every release. A custom image extending
+`barichello/godot-ci:4.7.1` with the toolchain pre-installed removes it.
+
+## What this spec is for
+
+A fresh Gitea agent session, given only this file, must be able to:
+
+1. Create a new repo.
+2. Add a `Dockerfile` and a publish workflow to it.
+3. Get the image published to that repo's Gitea package (container) registry.
+4. Report back the final image reference so `export-template.yml` here can be pointed at it.
+
+No other context from this repo should be required.
+
+## Scope: what goes in the image
+
+Everything `export-template.yml`'s "Update and install dependencies" step currently
+`apt-get install`s, unconditionally (all of it — see [Decisions](#decisions) below for why the
+narrower alternative was rejected):
+
+```
+libfontconfig1
+build-essential scons pkg-config xz-utils curl
+# Linux leg:
+libx11-dev libxcursor-dev libxinerama-dev libgl1-mesa-dev \
+  libasound2-dev libpulse-dev libudev-dev libxi-dev libxrandr-dev libwayland-dev
+# Windows leg:
+mingw-w64
+# + these two update-alternatives calls, baked in at image build time:
+update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix
+update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
+# conditionally, when ENABLE_BLENDER is on:
+blender
+```
+
+Current source of truth for this list: `.gitea/workflows/export-template.yml` in
+`haylan/Project4x`, "Update and install dependencies" step. **Re-check that step before building**
+— it may have drifted since this spec was written.
+
+### What stays out of the image
+
+- **The SCons object cache** (`.scons_cache`, restored via `actions/cache@v3` keyed on
+  `scons-${GODOT_VERSION}-${artifact_name}`). This is build _state_, not a package — baking it into
+  the image would pin it to image-build time instead of the actual release run, going stale
+  immediately.
+- **`nsis` / `osslsigncode`.** These do NOT belong in this image. They run in `release.yml`'s
+  `create-release` job, which is a plain `ubuntu-latest` runner — it does not use
+  `barichello/godot-ci` or any container at all. (Earlier drafts of this ticket assumed they were
+  part of the export container's toolchain; that was wrong. The NSIS installer has since been
+  removed entirely — see ADR 0001's superseded note — so only `osslsigncode`/`zip` remain in
+  `create-release`, and neither is expensive enough to justify its own image.)
+
+## Registry path and versioning
+
+- **Registry:** this Gitea instance's container/package registry (`git.arthurerlich.de`), scoped
+  to the new repo you create — Gitea packages are per-repo/owner, so the path is
+  `git.arthurerlich.de/haylan/<new-repo-name>`.
+- **Tag:** pin to the Godot version only, e.g. `git.arthurerlich.de/haylan/<new-repo-name>:4.7.1`.
+  This mirrors `barichello/godot-ci`'s own tag scheme and `Project4x`'s `GODOT_VERSION` env var. A
+  Godot version bump in `Project4x` means building and pushing a new tag here too — that coupling
+  is intentional, not an oversight.
+- Pick `<new-repo-name>` yourself (e.g. `godot-ci-toolchain` or similar); it isn't fixed by this
+  spec.
+
+## Auth: PAT required
+
+Confirmed for this Gitea instance: `GITEA_TOKEN` (the automatic per-run Actions token) **cannot**
+push to the package/container registry. The new repo's publish workflow needs a **Personal Access
+Token** stored as a repo secret (e.g. `REGISTRY_PUSH_PAT`).
+
+- **Scope needed:** `write:package` (add `read:package` too if the publish workflow ever needs to
+  pull the image back, e.g. to test it before tagging `latest`).
+- Generate it from the account that should own the published packages, under
+  Settings → Applications → Generate New Token, with only that scope checked.
+- Document the token's owner/scope in the new repo's README once created — this repo
+  (`Project4x`) has no record of it and shouldn't need one; the coupling is one-way
+  (`Project4x` only ever pulls the finished image, never pushes to it).
+
+## Dockerfile skeleton
+
+```dockerfile
+FROM barichello/godot-ci:4.7.1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libfontconfig1 \
+    build-essential scons pkg-config xz-utils curl \
+    libx11-dev libxcursor-dev libxinerama-dev libgl1-mesa-dev \
+    libasound2-dev libpulse-dev libudev-dev libxi-dev libxrandr-dev libwayland-dev \
+    mingw-w64 \
+    blender \
+  && update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix \
+  && update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix \
+  && rm -rf /var/lib/apt/lists/*
+```
+
+Notes for whoever builds this:
+
+- `blender` is baked in unconditionally here even though `Project4x`'s `ENABLE_BLENDER` flag is
+  currently `"false"` — the image can't cheaply branch on a flag it doesn't know about at build
+  time, and the package is small next to the rest of this list. If that turns out to add
+  meaningful image size/pull time for a project that never flips the flag on, split into a second
+  `-blender` tagged variant instead.
+- Combine into one `RUN` (as above) so it's a single image layer — no benefit to splitting these
+  across layers here.
+
+## Publish workflow skeleton
+
+New repo, `.gitea/workflows/publish.yml`:
+
+```yaml
+name: Publish godot-ci image
+
+on:
+  push:
+    branches: [main]
+    paths: [Dockerfile]
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to Gitea registry
+        run: echo "${{ secrets.REGISTRY_PUSH_PAT }}" | docker login git.arthurerlich.de -u <owner-username> --password-stdin
+
+      - name: Build and push
+        run: |
+          docker build -t git.arthurerlich.de/haylan/<new-repo-name>:4.7.1 .
+          docker push git.arthurerlich.de/haylan/<new-repo-name>:4.7.1
+```
+
+Fill in `<owner-username>` and `<new-repo-name>` once the repo exists.
+
+## How `Project4x` would then reference the new image
+
+One-line change in `.gitea/workflows/export-template.yml`:
+
+```diff
+- container: barichello/godot-ci:4.7.1
+ container: git.arthurerlich.de/haylan/<new-repo-name>:4.7.1
+```
+
+The "Update and install dependencies" step's `apt-get install` calls for the baked-in packages
+become redundant at that point (harmless no-ops — `apt-get install` on an already-installed
+package is a fast no-op, so removing them is a cleanup, not a correctness requirement) and can be
+trimmed in a follow-up ticket once the image is live and proven.
+
+## Decisions
+
+- **Bake-in scope:** everything in the current apt-get list, not a narrower "slow packages only"
+  subset — keeps the image and `export-template.yml`'s toolchain description in exact sync, so
+  there's only one place (this spec + the Dockerfile) to update when the list changes, instead of
+  maintaining a second smaller list that drifts from the first.
+- **Tag scheme:** pinned to `GODOT_VERSION` rather than tracking image/toolchain revisions
+  independently — simpler to reason about, and this repo's own SCons cache is already keyed the
+  same way, so a Godot bump already forces everything else to rebuild in lockstep.
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Shared parsing helpers for resolve-godot-version.sh and resolve-blender-url.sh.
+# Kept separate from the network calls so the parsing logic can be unit-tested
+# offline — see test-lib.sh.
+
+# Reads a GitHub releases API response (JSON array) on stdin, prints the newest
+# non-prerelease, non-draft tag with its "-stable" suffix stripped (e.g. "4.7.1").
+parse_latest_godot_tag() {
+  jq -r '[.[] | select(.prerelease == false and .draft == false)][0].tag_name' | sed 's/-stable$//'
+}
+
+# Reads the download.blender.org/release/ index HTML on stdin, prints the newest
+# "X.Y" minor-version directory name (no trailing slash), numerically not
+# lexicographically (so "4.10" correctly beats "4.2").
+parse_latest_blender_minor() {
+  grep -oE 'Blender[0-9]+\.[0-9]+/' | grep -oE '[0-9]+\.[0-9]+' | sort -t. -k1,1n -k2,2n -u | tail -1
+}
+
+# Reads a Blender minor-version release index HTML on stdin, prints the newest
+# linux-x64 tarball filename (ignores windows/mac assets and .sha256 sidecars).
+parse_latest_blender_tarball() {
+  grep -oE 'blender-[0-9.]+-linux-x64\.tar\.xz' | sort -V -u | tail -1
+}
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Prints the download URL of the newest stable Blender linux-x64 release tarball.
+# No official "latest stable" API exists for Blender (only the experimental-builds
+# API does) so this scrapes the release index — see docs/handoff for background.
+# Requires: curl.
+set -euo pipefail
+cd "$(dirname "$0")"
+. ./lib.sh
+
+base="https://download.blender.org/release/"
+minor=$(curl -fsSL "$base" | parse_latest_blender_minor)
+minor_url="${base}Blender${minor}/"
+file=$(curl -fsSL "$minor_url" | parse_latest_blender_tarball)
+
+echo "${minor_url}${file}"
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Prints the newest stable Godot Engine version (e.g. "4.7.1"), no "-stable" suffix.
+# Requires: curl, jq.
+set -euo pipefail
+cd "$(dirname "$0")"
+. ./lib.sh
+
+curl -fsSL https://api.github.com/repos/godotengine/godot/releases | parse_latest_godot_tag
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Offline smoke test for lib.sh's parsing logic (no network). Run: ./scripts/test-lib.sh
+set -euo pipefail
+cd "$(dirname "$0")"
+. ./lib.sh
+
+fail() { echo "FAIL: $1" >&2; exit 1; }
+
+# parse_latest_godot_tag: skips prereleases and drafts, strips "-stable"
+# (needs jq — present on the CI runner; skipped here if this machine lacks it)
+if command -v jq >/dev/null 2>&1; then
+  got=$(printf '%s' '[
+    {"tag_name":"4.8-rc1","prerelease":true,"draft":false},
+    {"tag_name":"4.7.1-stable","prerelease":false,"draft":false},
+    {"tag_name":"4.7-stable","prerelease":false,"draft":false},
+    {"tag_name":"4.8-dev-draft","prerelease":false,"draft":true}
+  ]' | parse_latest_godot_tag)
+  [ "$got" = "4.7.1" ] || fail "parse_latest_godot_tag: got '$got', want 4.7.1"
+else
+  echo "skip: parse_latest_godot_tag (jq not installed on this machine)" >&2
+fi
+
+# parse_latest_blender_minor: numeric sort, not lexicographic ("4.10" > "4.2")
+got=$(printf '%s' '<a href="Blender3.6/">Blender3.6/</a> <a href="Blender4.2/">Blender4.2/</a> <a href="Blender4.10/">Blender4.10/</a>' \
+  | parse_latest_blender_minor)
+[ "$got" = "4.10" ] || fail "parse_latest_blender_minor: got '$got', want 4.10"
+
+# parse_latest_blender_tarball: newest patch, ignores sha256 sidecars and other OSes
+got=$(printf '%s' '<a href="blender-4.2.1-linux-x64.tar.xz">x</a>
+<a href="blender-4.2.10-linux-x64.tar.xz">x</a>
+<a href="blender-4.2.10-linux-x64.tar.xz.sha256">x</a>
+<a href="blender-4.2.10-windows-x64.zip">x</a>' | parse_latest_blender_tarball)
+[ "$got" = "blender-4.2.10-linux-x64.tar.xz" ] || fail "parse_latest_blender_tarball: got '$got', want blender-4.2.10-linux-x64.tar.xz"
+
+echo "ok: scripts/lib.sh"