probe: stop filtering away the RUN output

The probe kept succeeding while its results vanished: the grep -vE plus tail -40
dropped the buildkit progress lines that carry the actual curl output. Keep the
full log, then print only the RUN lines with their #N prefix stripped.

.github/workflows/build-image-gitea.yml

@@ -0,0 +1,167 @@
+name: build image (gitea)
+
+# Publishes the runner toolchain image from the SELF-HOSTED Gitea runners, via
+# the in-cluster rootless buildkitd — the same path zegit's own images already
+# use (zegit .github/workflows/image-gitea.yml).
+#
+# Why this exists alongside build-image.yml: that workflow is GitHub-only, so
+# the image the Gitea runners are BUILT FROM could only be produced by GitHub.
+# When GitHub Actions became unavailable on 2026-08-29 that circularity bit —
+# the fix for a missing tool in this very image (openssl/jq, whose absence
+# blocked v0.0.134 and v0.0.135) could not be published on the forge the project
+# is moving to. This removes the dependency.
+#
+# build-image.yml's header says the runners cannot build images because rootless
+# buildah needs user namespaces the pod does not have. That is true of buildah
+# IN THE JOB, and irrelevant here: buildkitd is its own hardened workload in
+# buildkit-system, and this job only speaks to it over the network with
+# buildctl. Nothing is built inside the runner container.
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag, e.g. 12.11.1-ci6 (upstream runner version + ci revision)'
+        required: true
+        type: string
+  push:
+    branches: [master]
+    paths:
+      - 'Dockerfile'
+      - '.github/workflows/build-image-gitea.yml'
+
+env:
+  BUILDKIT_HOST: tcp://buildkitd.buildkit-system.svc.cluster.local:1234
+  BUILDKIT_VERSION: v0.32.2
+  REGISTRY: gitea.langenoja.se
+  IMAGE_PATH: zegit-zoo/zegit-forgejo-runner
+
+jobs:
+  build:
+    # Gitea-only: buildkitd is an in-cluster service that GitHub runners cannot
+    # reach. build-image.yml still covers the GitHub/ghcr.io side.
+    if: ${{ github.server_url != 'https://github.com' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          persist-credentials: false
+
+      # Workspace-local because the runner is non-root and cannot write to
+      # /usr/local/bin. The smoke test uses the FULL PATH deliberately:
+      # $GITHUB_PATH only takes effect for SUBSEQUENT steps, so a bare
+      # `buildctl --version` here would be command-not-found.
+      - name: Install buildctl
+        run: |
+          set -euo pipefail
+          case "$(uname -m)" in
+            x86_64) arch=amd64;;
+            aarch64) arch=arm64;;
+            *) echo "::error::unsupported arch $(uname -m)"; exit 1;;
+          esac
+          mkdir -p "$GITHUB_WORKSPACE/.bin"
+          curl -fsSL "https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-${arch}.tar.gz" \
+            | tar -xz -C "$GITHUB_WORKSPACE/.bin" --strip-components=1 bin/buildctl
+          echo "$GITHUB_WORKSPACE/.bin" >> "$GITHUB_PATH"
+          "$GITHUB_WORKSPACE/.bin/buildctl" --version
+
+      - name: buildkitd reachable
+        run: buildctl debug workers
+
+      # A master push VALIDATES that the Dockerfile still builds on both arches
+      # but publishes nothing; only an explicit dispatch names a tag and pushes.
+      # Guessing a tag from a branch push would silently mint versions.
+      - id: meta
+        run: |
+          set -euo pipefail
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "push=true"  >> "$GITHUB_OUTPUT"
+            echo "tag=${{ inputs.tag }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "push=false" >> "$GITHUB_OUTPUT"
+            echo "tag=ci-validation" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Refuse a publish that cannot complete, rather than reporting success
+      # having pushed nothing. A validation run needs no credentials.
+      # The Gitea Actions token cannot write packages; this needs an explicit
+      # package-write PAT (read:package is pull-only).
+      - name: Registry login
+        if: ${{ steps.meta.outputs.push == 'true' }}
+        env:
+          TOKEN: ${{ secrets.ZEGIT_PACKAGE_TOKEN }}
+        run: |
+          set -euo pipefail
+          if [ -z "${TOKEN:-}" ]; then
+            echo "::error::ZEGIT_PACKAGE_TOKEN is not set, so this dispatch would build an image and push nothing. Issue a package-write token and store it as this secret."
+            exit 1
+          fi
+          mkdir -p "$HOME/.docker"
+          printf '{"auths":{"%s":{"auth":"%s"}}}\n' \
+            "$REGISTRY" "$(printf 'zegit-zoo:%s' "$TOKEN" | base64 -w0)" \
+            > "$HOME/.docker/config.json"
+
+      - name: Build multi-arch
+        run: |
+          set -euo pipefail
+          REF="${REGISTRY}/${IMAGE_PATH}:${{ steps.meta.outputs.tag }}"
+          if [ "${{ steps.meta.outputs.push }}" = "true" ]; then
+            out="type=image,name=${REF},push=true"
+          else
+            out="type=cacheonly"
+          fi
+          buildctl build --progress=plain \
+            --frontend dockerfile.v0 \
+            --local context=. \
+            --local dockerfile=. \
+            --opt platform=linux/amd64,linux/arm64 \
+            --output "$out"
+
+      # The image is a TOOLCHAIN, so "it built" is not the property that matters
+      # — "it still contains what the gates shell out to" is. build-image.yml
+      # asserts that with `docker run`, which does not exist on these runners;
+      # buildkit can do it directly by building a stage FROM the published image
+      # whose RUN fails when a tool is absent. That is strictly better: buildkit
+      # runs it natively per platform, so a package that resolves on amd64 but
+      # not on arm64 cannot reach half the nodes unnoticed.
+      #
+      # openssl and jq are on this list because their absence blocked v0.0.134
+      # and v0.0.135: the release tag gate shells out to both to verify the
+      # Control Plane's signed quorum verdict, and a missing tool was reported
+      # as "verdict not signed" and "signature FAILED verification" — a tooling
+      # gap narrated as a security verdict. Extend this list whenever a gate
+      # grows a shell dependency; it only covers what someone remembered to add.
+      - name: Assert the published image carries the toolchain the gates need
+        if: ${{ steps.meta.outputs.push == 'true' }}
+        run: |
+          set -euo pipefail
+          REF="${REGISTRY}/${IMAGE_PATH}:${{ steps.meta.outputs.tag }}"
+          work=$(mktemp -d)
+          {
+            echo "FROM ${REF}"
+            echo 'RUN set -eu; \'
+            echo '    python3 -c "import yaml, sys; print(\"python3+yaml\", sys.version.split()[0])"; \'
+            echo '    for t in gcc node bash make curl tar openssl jq; do \'
+            echo '      command -v "$t" >/dev/null || { echo "MISSING: $t" >&2; exit 1; }; \'
+            echo '      echo "ok: $t"; \'
+            echo '    done; \'
+            echo '    test -x "$(git --exec-path)/git-http-backend" || { echo "MISSING: git-http-backend" >&2; exit 1; }; \'
+            echo '    echo "ok: git-http-backend"'
+          } > "$work/Dockerfile"
+          cat "$work/Dockerfile"
+          buildctl build --progress=plain \
+            --frontend dockerfile.v0 \
+            --local context="$work" \
+            --local dockerfile="$work" \
+            --opt platform=linux/amd64,linux/arm64 \
+            --output type=cacheonly
+          echo "✓ toolchain intact on linux/amd64 and linux/arm64"
+
+      - name: Summary
+        run: |
+          if [ "${{ steps.meta.outputs.push }}" = "true" ]; then
+            echo "published ${REGISTRY}/${IMAGE_PATH}:${{ steps.meta.outputs.tag }} (amd64+arm64), toolchain asserted on both"
+          else
+            echo "validation only — the Dockerfile builds on both arches; nothing published"
+          fi

.github/workflows/probe-buildkitd.yml

@@ -0,0 +1,66 @@
+name: probe registry from buildkitd
+
+# TEMPORARY probe branch — not for merge.
+#
+# The first probe ran in the RUNNER pod. buildkitd is a different pod in a
+# different namespace with its own NetworkPolicies, so those results cannot be
+# transferred: an UNREACHABLE there may be the runner's policy, not routing.
+# This runs the same probes THROUGH buildkitd, by building a stage whose RUN
+# does the curling — that egresses from the buildkitd pod, which is the network
+# position that actually has to push.
+#
+# It prints results and always exits 0: this is a measurement, not a gate.
+
+on:
+  workflow_dispatch:
+
+env:
+  BUILDKIT_HOST: tcp://buildkitd.buildkit-system.svc.cluster.local:1234
+  BUILDKIT_VERSION: v0.32.2
+
+jobs:
+  probe:
+    if: ${{ github.server_url != 'https://github.com' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Install buildctl
+        run: |
+          set -euo pipefail
+          case "$(uname -m)" in x86_64) arch=amd64;; aarch64) arch=arm64;; *) exit 1;; esac
+          mkdir -p "$GITHUB_WORKSPACE/.bin"
+          curl -fsSL "https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-${arch}.tar.gz" \
+            | tar -xz -C "$GITHUB_WORKSPACE/.bin" --strip-components=1 bin/buildctl
+          echo "$GITHUB_WORKSPACE/.bin" >> "$GITHUB_PATH"
+          "$GITHUB_WORKSPACE/.bin/buildctl" --version
+
+      - name: Probe from inside buildkitd
+        run: |
+          set -euo pipefail
+          work=$(mktemp -d)
+          {
+            echo 'FROM alpine:3.20'
+            echo 'RUN apk add --no-cache curl >/dev/null 2>&1 || true'
+            echo 'RUN set -u; \'
+            echo '  f="http=%{http_code} tls=%{ssl_verify_result} t=%{time_total}\\n"; \'
+            echo '  echo "== public name via cluster DNS =="; \'
+            echo '  curl -s -o /dev/null -w "$f" --max-time 8 https://gitea.langenoja.se/v2/ || echo UNREACHABLE; \'
+            echo '  echo "== resolves to =="; getent hosts gitea.langenoja.se || echo "(none)"; \'
+            echo '  echo "== istio-gateway external lb 10.40.0.10 =="; \'
+            echo '  curl -s -o /dev/null -w "$f" --max-time 8 --resolve gitea.langenoja.se:443:10.40.0.10 https://gitea.langenoja.se/v2/ || echo UNREACHABLE; \'
+            echo '  echo "== istio-gateway clusterip 10.43.1.114 =="; \'
+            echo '  curl -s -o /dev/null -w "$f" --max-time 8 --resolve gitea.langenoja.se:443:10.43.1.114 https://gitea.langenoja.se/v2/ || echo UNREACHABLE; \'
+            echo '  echo "== gitea http service in-cluster =="; \'
+            echo '  curl -s -o /dev/null -w "$f" --max-time 8 http://gitea-system-gitea-http.gitea-system.svc.cluster.local:3000/v2/ || echo UNREACHABLE; \'
+            echo '  echo "== gitea http service, headers =="; \'
+            echo '  curl -s -D- -o /dev/null --max-time 8 http://gitea-system-gitea-http.gitea-system.svc.cluster.local:3000/v2/ | head -5 || true'
+          } > "$work/Dockerfile"
+          cat "$work/Dockerfile"
+          buildctl build --progress=plain \
+            --frontend dockerfile.v0 \
+            --local context="$work" \
+            --local dockerfile="$work" \
+            --opt platform=linux/amd64 2>&1 | tee /tmp/probe.log | tail -80
+          echo "=== PROBE RESULTS ==="
+          grep -E "^#[0-9]+ [0-9.]+ " /tmp/probe.log | sed -E "s/^#[0-9]+ [0-9.]+ //" || echo "(no RUN output captured)"
+          echo "probe finished (results above, from buildkitd's network position)"

.github/workflows/probe-registry.yml

@@ -0,0 +1,47 @@
+name: probe registry endpoints
+
+# TEMPORARY probe branch — not for merge. Answers one question with evidence
+# instead of inference: which registry endpoints are reachable from a pod inside
+# this cluster, after 10.40.0.10 (the istio-gateway LoadBalancer EXTERNAL
+# address) turned out to be connection-refused from the pod network while
+# answering fine from a LAN workstation.
+
+on:
+  workflow_dispatch:
+
+jobs:
+  probe:
+    if: ${{ github.server_url != 'https://github.com' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Probe candidate registry endpoints
+        run: |
+          set -u
+          fmt='http=%{http_code} tls=%{ssl_verify_result} t=%{time_total}\n'
+
+          echo "--- public name via cluster DNS (the path that works today) ---"
+          printf '%-46s ' 'gitea.langenoja.se:443'
+          curl -s -o /dev/null -w "$fmt" --max-time 8 https://gitea.langenoja.se/v2/ \
+            || echo "UNREACHABLE"
+
+          echo "--- istio-gateway EXTERNAL lb ip (what #949 pinned) ---"
+          printf '%-46s ' '10.40.0.10:443'
+          curl -s -o /dev/null -w "$fmt" --max-time 8 \
+            --resolve gitea.langenoja.se:443:10.40.0.10 https://gitea.langenoja.se/v2/ \
+            || echo "UNREACHABLE"
+
+          echo "--- istio-gateway CLUSTER ip (candidate) ---"
+          printf '%-46s ' '10.43.1.114:443'
+          curl -s -o /dev/null -w "$fmt" --max-time 8 \
+            --resolve gitea.langenoja.se:443:10.43.1.114 https://gitea.langenoja.se/v2/ \
+            || echo "UNREACHABLE"
+
+          echo "--- gitea http service, in-cluster, plain http ---"
+          printf '%-46s ' 'gitea-system-gitea-http:3000'
+          curl -s -o /dev/null -w "$fmt" --max-time 8 \
+            http://gitea-system-gitea-http.gitea-system.svc.cluster.local:3000/v2/ \
+            || echo "UNREACHABLE"
+
+          echo "--- what this pod resolves the public name to ---"
+          getent hosts gitea.langenoja.se || echo "(getent failed)"