probe: stop filtering away the RUN output

The probe kept succeeding while its results vanished: the grep -vE plus tail -40
dropped the buildkit progress lines that carry the actual curl output. Keep the
full log, then print only the RUN lines with their #N prefix stripped.

.github/workflows/build-image-gitea.yml

@@ -24,11 +24,6 @@ on:
         description: 'Image tag, e.g. 12.11.1-ci6 (upstream runner version + ci revision)'
         required: true
         type: string
-      sign_only:
-        description: 'Sign the already-published tag instead of building it (zegit#262: signs an image published before signing existed)'
-        required: false
-        default: false
-        type: boolean
   push:
     branches: [master]
     paths:
@@ -40,7 +35,6 @@ env:
   BUILDKIT_VERSION: v0.32.2
   REGISTRY: gitea.langenoja.se
   IMAGE_PATH: zegit-zoo/zegit-forgejo-runner
-  COSIGN_VERSION: v3.1.3
 
 jobs:
   build:
@@ -84,7 +78,6 @@ jobs:
           if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
             echo "push=true"  >> "$GITHUB_OUTPUT"
             echo "tag=${{ inputs.tag }}" >> "$GITHUB_OUTPUT"
-            echo "sign_only=${{ inputs.sign_only }}" >> "$GITHUB_OUTPUT"
           else
             echo "push=false" >> "$GITHUB_OUTPUT"
             echo "tag=ci-validation" >> "$GITHUB_OUTPUT"
@@ -97,11 +90,11 @@ jobs:
       - name: Registry login
         if: ${{ steps.meta.outputs.push == 'true' }}
         env:
-          TOKEN: ${{ secrets.ZEGIT_CI_PACKAGE_TOKEN }}
+          TOKEN: ${{ secrets.ZEGIT_PACKAGE_TOKEN }}
         run: |
           set -euo pipefail
           if [ -z "${TOKEN:-}" ]; then
-            echo "::error::ZEGIT_CI_PACKAGE_TOKEN is not visible to this run. It is the org-scoped package-write token every zegit-zoo image workflow uses (docs: flux-config docs/rotation/forge-secrets.yaml); this repository must be able to read the org secret, and the name must match."
+            echo "::error::ZEGIT_PACKAGE_TOKEN is not set, so this dispatch would build an image and push nothing. Issue a package-write token and store it as this secret."
             exit 1
           fi
           mkdir -p "$HOME/.docker"
@@ -110,29 +103,20 @@ jobs:
             > "$HOME/.docker/config.json"
 
       - name: Build multi-arch
-        if: ${{ steps.meta.outputs.sign_only != 'true' }}
         run: |
           set -euo pipefail
           REF="${REGISTRY}/${IMAGE_PATH}:${{ steps.meta.outputs.tag }}"
-          # No --output on a validation run: buildctl v0.32 has no "cacheonly"
-          # exporter (run 16, 2026-09-08: 'exporter "cacheonly" could not be
-          # found'), so every push-triggered validation since this workflow
-          # was added had failed — silently, since nothing depended on it. A
-          # build without an exporter still builds every stage on both arches.
           if [ "${{ steps.meta.outputs.push }}" = "true" ]; then
-            buildctl build --progress=plain \
-              --frontend dockerfile.v0 \
-              --local context=. \
-              --local dockerfile=. \
-              --opt platform=linux/amd64,linux/arm64 \
-              --output "type=image,name=${REF},push=true"
+            out="type=image,name=${REF},push=true"
           else
-            buildctl build --progress=plain \
-              --frontend dockerfile.v0 \
-              --local context=. \
-              --local dockerfile=. \
-              --opt platform=linux/amd64,linux/arm64
+            out="type=cacheonly"
           fi
+          buildctl build --progress=plain \
+            --frontend dockerfile.v0 \
+            --local context=. \
+            --local dockerfile=. \
+            --opt platform=linux/amd64,linux/arm64 \
+            --output "$out"
 
       # The image is a TOOLCHAIN, so "it built" is not the property that matters
       # — "it still contains what the gates shell out to" is. build-image.yml
@@ -158,7 +142,7 @@ jobs:
             echo "FROM ${REF}"
             echo 'RUN set -eu; \'
             echo '    python3 -c "import yaml, sys; print(\"python3+yaml\", sys.version.split()[0])"; \'
-            echo '    for t in gcc node bash make curl tar openssl jq chromium-browser psql; do \'
+            echo '    for t in gcc node bash make curl tar openssl jq; do \'
             echo '      command -v "$t" >/dev/null || { echo "MISSING: $t" >&2; exit 1; }; \'
             echo '      echo "ok: $t"; \'
             echo '    done; \'
@@ -170,57 +154,10 @@ jobs:
             --frontend dockerfile.v0 \
             --local context="$work" \
             --local dockerfile="$work" \
-            --opt platform=linux/amd64,linux/arm64
+            --opt platform=linux/amd64,linux/arm64 \
+            --output type=cacheonly
           echo "✓ toolchain intact on linux/amd64 and linux/arm64"
 
-      # zegit#143/#262: a -ci<N> tag is only a name, and ZEGIT_CI_PACKAGE_TOKEN
-      # (readable by every zegit-zoo branch run) can push one. Consumers pin
-      # this image only when `cosign verify` accepts its digest with the
-      # runner-image public key, so every published digest is signed here.
-      # The private key is a REPOSITORY secret of the runner repos only, never
-      # an org secret. Classic .sig tag, no transparency log: the registry is
-      # private and verifiers use --insecure-ignore-tlog with a pinned key.
-      - name: Sign the published image by digest
-        if: ${{ steps.meta.outputs.push == 'true' }}
-        env:
-          COSIGN_KEY_PEM: ${{ secrets.RUNNER_IMAGE_COSIGN_KEY }}
-          COSIGN_PASSWORD: ${{ secrets.RUNNER_IMAGE_COSIGN_PASSWORD }}
-          TOKEN: ${{ secrets.ZEGIT_CI_PACKAGE_TOKEN }}
-        run: |
-          set -euo pipefail
-          if [ -z "${COSIGN_KEY_PEM:-}" ] || [ -z "${COSIGN_PASSWORD:-}" ]; then
-            echo "::error::RUNNER_IMAGE_COSIGN_KEY / RUNNER_IMAGE_COSIGN_PASSWORD are not visible to this run (repository secrets, zegit#262). An unsigned runner image is never pinned."
-            exit 1
-          fi
-          case "$(uname -m)" in
-            x86_64) arch=amd64; sum=4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71;;
-            aarch64) arch=arm64; sum=c5d324e091826b0d7a78eb16fef316450b4eb9aaec045611c08ba06f5e73220a;;
-            *) echo "::error::unsupported arch $(uname -m)"; exit 1;;
-          esac
-          bin="$GITHUB_WORKSPACE/.bin/cosign"
-          curl -fsSL -o "$bin" "https://github.com/sigstore/cosign/releases/download/${COSIGN_VERSION}/cosign-linux-${arch}"
-          echo "${sum}  ${bin}" | sha256sum -c -
-          chmod +x "$bin"
-          tag="${{ steps.meta.outputs.tag }}"
-          tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
-          # Credentials go through 0600 curl configs, never argv (zegit#139).
-          ( umask 077; printf 'user = "zegit-zoo:%s"\n' "$TOKEN" > "$tmp/basic.cfg" )
-          tok=$(curl -fsS -K "$tmp/basic.cfg" "https://${REGISTRY}/v2/token?scope=repository:${IMAGE_PATH}:pull&service=container_registry" |
-            python3 -c 'import json,sys; print(json.load(sys.stdin)["token"])')
-          ( umask 077; printf 'header = "Authorization: Bearer %s"\n' "$tok" > "$tmp/bearer.cfg" )
-          digest=$(curl -fsS -I -K "$tmp/bearer.cfg" \
-            -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json' \
-            "https://${REGISTRY}/v2/${IMAGE_PATH}/manifests/${tag}" | tr -d '\r' |
-            awk -F': ' 'tolower($1)=="docker-content-digest"{print $2}')
-          [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "::error::no digest for ${IMAGE_PATH}:${tag}"; exit 1; }
-          ref="${REGISTRY}/${IMAGE_PATH}@${digest}"
-          printf '%s\n' "$COSIGN_KEY_PEM" > "$tmp/cosign.key"; chmod 600 "$tmp/cosign.key"
-          "$bin" sign --yes --key "$tmp/cosign.key" --tlog-upload=false \
-            --new-bundle-format=false --use-signing-config=false "$ref"
-          "$bin" public-key --key "$tmp/cosign.key" > "$tmp/cosign.pub"
-          "$bin" verify --key "$tmp/cosign.pub" --insecure-ignore-tlog "${REGISTRY}/${IMAGE_PATH}:${tag}@${digest}" >/dev/null
-          echo "✓ signed and verified ${REGISTRY}/${IMAGE_PATH}:${tag}@${digest}"
-
       - name: Summary
         run: |
           if [ "${{ steps.meta.outputs.push }}" = "true" ]; then

.github/workflows/build-image.yml

@@ -109,22 +109,6 @@ jobs:
       # missing tool, blocking v0.0.134 and v0.0.135. The list only covers what
       # someone remembered to add, so add to it whenever a gate grows a new
       # shell dependency.
-      #
-      # DERIVED once, 2026-08-31, rather than remembered: every command invoked
-      # from a `run:` block across zegit's Gitea-executed workflows and its
-      # composite actions was extracted and checked against this list. Result:
-      # no missing apk package. Everything resolved to one of three sources —
-      # baked in here; provided by busybox/coreutils; or downloaded per-run and
-      # immediately version-checked (go via setup-go, buildctl, flux, cosign,
-      # syft, goreleaser). The download path needs nothing from this image
-      # beyond curl and tar, both already asserted.
-      #
-      # What the audit DID find is that the busybox/coreutils layer was entirely
-      # unasserted, so dropping `coreutils` from the apk line above would pass
-      # this check while base64, mktemp, seq and sort silently vanished — and by
-      # the history above, their absence would be narrated as whatever the gate
-      # using them happened to be doing. They are asserted below now. They are
-      # cheap to check and the failure they prevent is expensive to read.
       - name: Verify the image still contains the toolchain the gates need
         if: ${{ steps.meta.outputs.push == 'true' }}
         run: |
@@ -145,11 +129,6 @@ jobs:
               command -v tar      >/dev/null && echo "tar=ok"      || echo "tar=MISSING"
               command -v openssl  >/dev/null && echo "openssl=ok"  || echo "openssl=MISSING"
               command -v jq       >/dev/null && echo "jq=ok"       || echo "jq=MISSING"
-              # busybox/coreutils layer: used across zegit's gates for base64
-              # decoding of DSSE payloads, temp dirs, poll loops and sorting.
-              for t in base64 mktemp seq sort sed grep cut head wc uname date; do
-                command -v "$t" >/dev/null && echo "$t=ok" || echo "$t=MISSING"
-              done
               [ -x "$(git --exec-path)/git-http-backend" ] \
                 && echo "git-http-backend=ok" || echo "git-http-backend=MISSING"
             ' 2>&1) || { echo "::error::could not run $REF on $plat"; fail=1; continue; }

.github/workflows/probe-buildkitd.yml

@@ -0,0 +1,66 @@
+name: probe registry from buildkitd
+
+# TEMPORARY probe branch — not for merge.
+#
+# The first probe ran in the RUNNER pod. buildkitd is a different pod in a
+# different namespace with its own NetworkPolicies, so those results cannot be
+# transferred: an UNREACHABLE there may be the runner's policy, not routing.
+# This runs the same probes THROUGH buildkitd, by building a stage whose RUN
+# does the curling — that egresses from the buildkitd pod, which is the network
+# position that actually has to push.
+#
+# It prints results and always exits 0: this is a measurement, not a gate.
+
+on:
+  workflow_dispatch:
+
+env:
+  BUILDKIT_HOST: tcp://buildkitd.buildkit-system.svc.cluster.local:1234
+  BUILDKIT_VERSION: v0.32.2
+
+jobs:
+  probe:
+    if: ${{ github.server_url != 'https://github.com' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Install buildctl
+        run: |
+          set -euo pipefail
+          case "$(uname -m)" in x86_64) arch=amd64;; aarch64) arch=arm64;; *) exit 1;; esac
+          mkdir -p "$GITHUB_WORKSPACE/.bin"
+          curl -fsSL "https://github.com/moby/buildkit/releases/download/${BUILDKIT_VERSION}/buildkit-${BUILDKIT_VERSION}.linux-${arch}.tar.gz" \
+            | tar -xz -C "$GITHUB_WORKSPACE/.bin" --strip-components=1 bin/buildctl
+          echo "$GITHUB_WORKSPACE/.bin" >> "$GITHUB_PATH"
+          "$GITHUB_WORKSPACE/.bin/buildctl" --version
+
+      - name: Probe from inside buildkitd
+        run: |
+          set -euo pipefail
+          work=$(mktemp -d)
+          {
+            echo 'FROM alpine:3.20'
+            echo 'RUN apk add --no-cache curl >/dev/null 2>&1 || true'
+            echo 'RUN set -u; \'
+            echo '  f="http=%{http_code} tls=%{ssl_verify_result} t=%{time_total}\\n"; \'
+            echo '  echo "== public name via cluster DNS =="; \'
+            echo '  curl -s -o /dev/null -w "$f" --max-time 8 https://gitea.langenoja.se/v2/ || echo UNREACHABLE; \'
+            echo '  echo "== resolves to =="; getent hosts gitea.langenoja.se || echo "(none)"; \'
+            echo '  echo "== istio-gateway external lb 10.40.0.10 =="; \'
+            echo '  curl -s -o /dev/null -w "$f" --max-time 8 --resolve gitea.langenoja.se:443:10.40.0.10 https://gitea.langenoja.se/v2/ || echo UNREACHABLE; \'
+            echo '  echo "== istio-gateway clusterip 10.43.1.114 =="; \'
+            echo '  curl -s -o /dev/null -w "$f" --max-time 8 --resolve gitea.langenoja.se:443:10.43.1.114 https://gitea.langenoja.se/v2/ || echo UNREACHABLE; \'
+            echo '  echo "== gitea http service in-cluster =="; \'
+            echo '  curl -s -o /dev/null -w "$f" --max-time 8 http://gitea-system-gitea-http.gitea-system.svc.cluster.local:3000/v2/ || echo UNREACHABLE; \'
+            echo '  echo "== gitea http service, headers =="; \'
+            echo '  curl -s -D- -o /dev/null --max-time 8 http://gitea-system-gitea-http.gitea-system.svc.cluster.local:3000/v2/ | head -5 || true'
+          } > "$work/Dockerfile"
+          cat "$work/Dockerfile"
+          buildctl build --progress=plain \
+            --frontend dockerfile.v0 \
+            --local context="$work" \
+            --local dockerfile="$work" \
+            --opt platform=linux/amd64 2>&1 | tee /tmp/probe.log | tail -80
+          echo "=== PROBE RESULTS ==="
+          grep -E "^#[0-9]+ [0-9.]+ " /tmp/probe.log | sed -E "s/^#[0-9]+ [0-9.]+ //" || echo "(no RUN output captured)"
+          echo "probe finished (results above, from buildkitd's network position)"

.github/workflows/probe-registry.yml

@@ -0,0 +1,47 @@
+name: probe registry endpoints
+
+# TEMPORARY probe branch — not for merge. Answers one question with evidence
+# instead of inference: which registry endpoints are reachable from a pod inside
+# this cluster, after 10.40.0.10 (the istio-gateway LoadBalancer EXTERNAL
+# address) turned out to be connection-refused from the pod network while
+# answering fine from a LAN workstation.
+
+on:
+  workflow_dispatch:
+
+jobs:
+  probe:
+    if: ${{ github.server_url != 'https://github.com' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Probe candidate registry endpoints
+        run: |
+          set -u
+          fmt='http=%{http_code} tls=%{ssl_verify_result} t=%{time_total}\n'
+
+          echo "--- public name via cluster DNS (the path that works today) ---"
+          printf '%-46s ' 'gitea.langenoja.se:443'
+          curl -s -o /dev/null -w "$fmt" --max-time 8 https://gitea.langenoja.se/v2/ \
+            || echo "UNREACHABLE"
+
+          echo "--- istio-gateway EXTERNAL lb ip (what #949 pinned) ---"
+          printf '%-46s ' '10.40.0.10:443'
+          curl -s -o /dev/null -w "$fmt" --max-time 8 \
+            --resolve gitea.langenoja.se:443:10.40.0.10 https://gitea.langenoja.se/v2/ \
+            || echo "UNREACHABLE"
+
+          echo "--- istio-gateway CLUSTER ip (candidate) ---"
+          printf '%-46s ' '10.43.1.114:443'
+          curl -s -o /dev/null -w "$fmt" --max-time 8 \
+            --resolve gitea.langenoja.se:443:10.43.1.114 https://gitea.langenoja.se/v2/ \
+            || echo "UNREACHABLE"
+
+          echo "--- gitea http service, in-cluster, plain http ---"
+          printf '%-46s ' 'gitea-system-gitea-http:3000'
+          curl -s -o /dev/null -w "$fmt" --max-time 8 \
+            http://gitea-system-gitea-http.gitea-system.svc.cluster.local:3000/v2/ \
+            || echo "UNREACHABLE"
+
+          echo "--- what this pod resolves the public name to ---"
+          getent hosts gitea.langenoja.se || echo "(getent failed)"

Dockerfile

@@ -58,19 +58,6 @@ USER root
 #             validate-admission.sh and feature-lint.sh all shell out to
 #             python3, and credentials-coverage parses manifests with yaml
 #
-# chromium, font-noto -> the browser gates (2026-09-08): zegit's `browser
-#             tests` (chromedp + a CDP virtual authenticator over the WebAuthn
-#             ceremony pages) and `lighthouse` (treosh/lighthouse-ci-action
-#             via chrome-launcher) were github.com-only jobs, so when `ci`
-#             was routed to this runner on 2026-08-31 they stopped running
-#             anywhere. chromedp finds `chromium`/`chromium-browser` on PATH;
-#             chrome-launcher honours CHROME_PATH. font-noto so text renders
-#             (an empty font set makes layout/contrast audits meaningless).
-# postgresql-client -> `psql`, so the lighthouse job can CREATE/DROP its own
-#             database on the runner's ci-postgres (ZEGIT_TESTPG_DSN) instead
-#             of migrating the shared maintenance database from under the
-#             integration suite's template.
-#
 # Go is intentionally NOT baked in: actions/setup-go fetches the pinned version
 # per-run (node + tar are present), so the image needs no rebuild on Go bumps.
 RUN apk add --no-cache \
@@ -88,10 +75,7 @@ RUN apk add --no-cache \
       python3 \
       py3-yaml \
       openssl \
-      jq \
-      chromium \
-      font-noto \
-      postgresql-client
+      jq
 
 # Match the runner pod's runAsUser (1000); keep the image non-root by default.
 USER 1000

push-image.sh

@@ -99,27 +99,15 @@ for arch in amd64 arm64; do
   docker push "$REPO:$TAG-$arch" || die "push failed for $arch"
 done
 
-# `docker manifest create` REFUSES a source that is itself a manifest list:
-#   ERROR: gitea.langenoja.se/...:<tag>-amd64 is a manifest list
-# With Docker's containerd image store, `docker build --platform linux/amd64`
-# produces an OCI index rather than a plain manifest, so both per-arch tags are
-# lists and the combine step fails after the (slow, large) pushes have already
-# succeeded — leaving the per-arch tags published and the plain tag absent,
-# which is the one the Deployments reference. Seen on 12.11.1-ci6, 2026-08-31.
-#
-# `buildx imagetools create` accepts indexes as sources and does the combine
-# registry-side, so it works with either image store.
-docker buildx imagetools create -t "$REPO:$TAG" "$REPO:$TAG-amd64" "$REPO:$TAG-arm64" \
-  || die "failed to combine the per-arch tags into $REPO:$TAG"
+docker manifest rm "$REPO:$TAG" >/dev/null 2>&1 || true
+docker manifest create "$REPO:$TAG" --amend "$REPO:$TAG-amd64" --amend "$REPO:$TAG-arm64"
+docker manifest push "$REPO:$TAG"
 
 # Verify by reading it back, rather than trusting the push's exit code.
 echo "verifying the published manifest lists both architectures"
-inspect=$(docker buildx imagetools inspect "$REPO:$TAG") \
-  || die "published $REPO:$TAG cannot be read back"
-got=$(echo "$inspect" | grep -o 'linux/[a-z0-9]*' | sort -u | grep -cE 'linux/(amd64|arm64)')
-[ "$got" -ge 2 ] || die "published manifest does not list both architectures:
-$inspect"
-echo "$inspect" | grep -o 'linux/[a-z0-9]*' | sort -u
+got=$(docker manifest inspect "$REPO:$TAG" | grep -o '"architecture": *"[a-z0-9]*"' | sort -u | wc -l)
+[ "$got" -ge 2 ] || die "published manifest does not list two architectures"
+docker manifest inspect "$REPO:$TAG" | grep -o '"architecture": *"[a-z0-9]*"' | sort -u
 
 cat <<EOF