From 00406c1e2d472e467f808d803e5282fd0231468c Mon Sep 17 00:00:00 2001 From: "atomicbackspace-agent[bot]" <330986079+atomicbackspace-agent[bot]@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:31:02 +0100 Subject: [PATCH] fix: build the runner with a schema that accepts Gitea 28's matrix jobs Gitea 28 adds job-index and job-total under strategy in the workflow it sends for a matrix job, and forgejo-runner's schema refuses them, so every matrix job dies before its first step (zegit image-gitea.yml run 23992, the v0.1.52 tag build). No forgejo-runner release accepts them (checked through 13.2.1). Build the runner from v12.11.1 at its commit with a one-file schema patch and two tests, one of them the real payload of that failed job; the build runs them. Bases are pinned by digest. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018wWyCiV27KmNSauYTdLyp7 --- .github/workflows/build-image-gitea.yml | 1 + Dockerfile | 25 +- ...ept-gitea28-strategy-job-index-total.patch | 687 ++++++++++++++++++ 3 files changed, 712 insertions(+), 1 deletion(-) create mode 100644 patches/0001-accept-gitea28-strategy-job-index-total.patch diff --git a/.github/workflows/build-image-gitea.yml b/.github/workflows/build-image-gitea.yml index 208ee85..0c27da4 100644 --- a/.github/workflows/build-image-gitea.yml +++ b/.github/workflows/build-image-gitea.yml @@ -33,6 +33,7 @@ on: branches: [master] paths: - 'Dockerfile' + - 'patches/**' - '.github/workflows/build-image-gitea.yml' env: diff --git a/Dockerfile b/Dockerfile index 9e338c3..a202fb2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,27 @@ # forgejo-runner-system -- github-registry-credentials for ghcr.io. An earlier # version of this comment claimed the package was public and needed no secret; # that was wrong for the Gitea registry too, which is private. -FROM code.forgejo.org/forgejo/runner:12.11.1 +# The runner binary is built from upstream v12.11.1 with one patch: Gitea 28 adds +# job-index and job-total under `strategy` in the workflow it sends for a matrix +# job, and the stock schema refuses them ("Unknown Property job-total ... the +# workflow file is not usable"), so every matrix job dies before its first step. +# No forgejo-runner release accepts them (checked through 13.2.1). Drop this stage +# when upstream does, or when the runner is replaced. +FROM --platform=$BUILDPLATFORM data.forgejo.org/oci/golang:1.25-alpine3.23@sha256:27b3069b25647beacd3745b0b2c83e2f9b8d3935455a32cd4531236ee1c960e0 AS runner-build +ARG TARGETOS +ARG TARGETARCH +RUN apk add --no-cache git +ADD --keep-git-dir=false https://code.forgejo.org/forgejo/runner.git#8e01d43cd27e0732a1cff664996a92739472d42b /srv +COPY patches/ /patches/ +WORKDIR /srv +RUN git apply --verbose /patches/*.patch +# The patch's own tests, so a build cannot ship a patch that stopped working. +RUN go test -count=1 ./act/schema/ ./internal/app/run/ -run 'StrategyJobIndexAndTotal|Gitea28MatrixJob' +RUN CGO_ENABLED=0 GOOS="$TARGETOS" GOARCH="$TARGETARCH" go build -trimpath -tags 'netgo osusergo' \ + -ldflags '-s -w -X code.forgejo.org/forgejo/runner/v12/internal/pkg/ver.version=v12.11.1' \ + -o /out/forgejo-runner . + +FROM code.forgejo.org/forgejo/runner:12.11.1@sha256:a1a2885261a0b8337d4791dd6949651117206741ba66423c24a57c107a64d4a8 USER root @@ -105,5 +125,8 @@ RUN set -eux; \ echo "brotli-libs ${v}"; \ [ "$(apk version -t "${v}" 1.2.1-r0)" != "<" ] || { echo "brotli-libs ${v} is older than 1.2.1-r0 (CVE-2026-111497)"; exit 1; } +# The patched binary replaces the stock one (see the runner-build stage). +COPY --from=runner-build /out/forgejo-runner /bin/forgejo-runner + # Match the runner pod's runAsUser (1000); keep the image non-root by default. USER 1000 diff --git a/patches/0001-accept-gitea28-strategy-job-index-total.patch b/patches/0001-accept-gitea28-strategy-job-index-total.patch new file mode 100644 index 0000000..96166b5 --- /dev/null +++ b/patches/0001-accept-gitea28-strategy-job-index-total.patch @@ -0,0 +1,687 @@ +diff --git a/act/schema/schema_test.go b/act/schema/schema_test.go +index 14daa46..6be5075 100644 +--- a/act/schema/schema_test.go ++++ b/act/schema/schema_test.go +@@ -571,3 +571,30 @@ jobs: + }) + } + } ++ ++func TestSchema_StrategyJobIndexAndTotal(t *testing.T) { ++ for _, value := range []string{"1", "${{ vars.N }}"} { ++ var node yaml.Node ++ err := yaml.Unmarshal([]byte(` ++on: [push] ++jobs: ++ test1: ++ runs-on: docker ++ strategy: ++ matrix: ++ image: [a] ++ job-index: `+value+` ++ job-total: `+value+` ++ steps: ++ - run: echo All good! ++`), &node) ++ if !assert.NoError(t, err) { ++ return ++ } ++ err = (&Node{ ++ Definition: "workflow-root", ++ Schema: GetWorkflowSchema(), ++ }).UnmarshalYAML(&node) ++ assert.NoError(t, err, value) ++ } ++} +diff --git a/act/schema/workflow_schema.json b/act/schema/workflow_schema.json +index 67ffbde..99bdcc8 100644 +--- a/act/schema/workflow_schema.json ++++ b/act/schema/workflow_schema.json +@@ -1573,7 +1573,15 @@ + "type": "max-parallel", + "description": "The maximum number of jobs that can run simultaneously when using a matrix job strategy. By default, GitHub will maximize the number of jobs run in parallel depending on runner availability." + }, +- "matrix": "matrix" ++ "matrix": "matrix", ++ "job-index": { ++ "type": "max-parallel", ++ "description": "Set by Gitea 28 on each job of an expanded matrix; the runner ignores it." ++ }, ++ "job-total": { ++ "type": "max-parallel", ++ "description": "Set by Gitea 28 on each job of an expanded matrix; the runner ignores it." ++ } + } + } + }, +diff --git a/internal/app/run/testdata/gitea28-matrix-job.yml b/internal/app/run/testdata/gitea28-matrix-job.yml +new file mode 100644 +index 0000000..6267e24 +--- /dev/null ++++ b/internal/app/run/testdata/gitea28-matrix-job.yml +@@ -0,0 +1,591 @@ ++name: image-gitea ++'on': ++ push: ++ branches: ["claude/**"] # PoC trigger; tags come in P2 ++ tags: ["v*"] ++ workflow_dispatch: ++env: ++ BUILDKIT_HOST: tcp://buildkitd.buildkit-system.svc.cluster.local:1234 ++ BUILDKIT_VERSION: v0.32.2 ++ REGISTRY: gitea.langenoja.se ++ # zegit publishes under a registry owner of its own, written only by the ++ # operator and the restricted CI user zegit-ci (RUNBOOK-TENANT-PACKAGES.md ++ # §3 option A). The Control Plane's image_path_prefix for zegit-zoo/zegit is ++ # ${REGISTRY}/${IMAGE_PATH}/, so it signs nothing anywhere else. Releases ++ # before this owner are under zegit-zoo/zegit; scripts/image-path-of.sh ++ # reads a tag's IMAGE_PATH from that tag's own copy of this file. ++ IMAGE_PATH: zegit-packages/zegit ++ # The account ZEGIT_CI_PACKAGE_TOKEN is minted on. Gitea authenticates the ++ # token's own account whatever name is sent (RUNBOOK-TENANT-PACKAGES.md §4), ++ # so the login names the owner it actually is. ++ REGISTRY_USER: zegit-ci ++jobs: ++ build: ++ name: build z-gateway-server (buildkit) ++ runs-on: ubuntu-latest ++ if: always() ++ steps: ++ - if: ${{ startsWith(github.ref, 'refs/tags/v') }} ++ name: End processes left behind by earlier jobs (zegit#137) ++ run: | ++ /usr/bin/python3 -I -S - --job-env <<'QUARANTINE' ++ #!/usr/bin/env python3 ++ """runner-quarantine — end every process a previous job left behind (zegit#137). ++ ++ The self-hosted Forgejo runner is host-mode: every job a runner pod takes ++ runs as the same uid in the same process namespace. A branch job can leave a ++ detached process behind (`setsid nohup implant &`) that outlives its job and ++ reads /proc//environ and /proc//cmdline of every later job on the ++ pod — the signer tokens of a release build included. The lasting fix is an ++ ephemeral, repository-scoped runner for the signing jobs (operator steps in ++ documentation/CI-ON-GITEA.md §11); this is the in-job stopgap. ++ ++ It runs as the FIRST step of every job that holds a signing credential — ++ before actions/checkout — and from the workflow text, not the workspace: the ++ workflow file comes from the server, while the checkout (and anything run ++ from it) is writable by the very process this ends. Each such step is ++ `/usr/bin/python3 -I -S - --job-env <<'QUARANTINE'` followed by this file ++ verbatim (-I -S: no user site, no usercustomize/sitecustomize, no PYTHON* ++ environment from the shared $HOME). scripts/workflow-policy.py checks that ++ every copy is byte-identical to this file; `workflow-policy.py ++ --print-quarantine-step` prints the step to paste. ++ ++ --job-env then gives the job a private HOME under RUNNER_TEMP (0700, an ++ empty .gitconfig) and exports HOME, XDG_CONFIG_HOME, XDG_CACHE_HOME, ++ GIT_CONFIG_GLOBAL and GIT_CONFIG_NOSYSTEM=1 through GITHUB_ENV, so no git ++ call of the job (actions/checkout's included) reads a ~/.gitconfig ++ (core.fsmonitor, core.hooksPath) or other dotfile an earlier job planted in ++ the runner's shared $HOME. ++ ++ A runner is capacity-1, so while this job runs nothing else of this uid ++ belongs on the pod except this job's own process ancestry (act_runner and the ++ step shell). Every other process of this uid is stopped (SIGSTOP, so it can ++ not fork away while we look), then killed (SIGKILL), and the scan repeats ++ until nothing is left. A survivor after ROUNDS rounds fails the step. ++ ++ Each process ended is named on a ::warning:: line by pid, name and parent — ++ never its command line, which may itself hold a credential. ++ ++ Refuses to run against the real /proc anywhere but a Gitea/Forgejo Actions ++ job: on a workstation "every other process of my uid" is the whole session. ++ --proc points it at a fixture tree for the tests (scripts/test-workflow-policy.sh). ++ """ ++ import argparse ++ import os ++ import signal ++ import sys ++ import tempfile ++ import time ++ ++ ROUNDS = 10 ++ ++ ++ def read_status(proc, pid): ++ """Return (name, ppid, uid, state) or None when the process is gone.""" ++ try: ++ with open(os.path.join(proc, pid, "status")) as f: ++ text = f.read() ++ except OSError: ++ return None ++ fields = {} ++ for line in text.splitlines(): ++ key, _, value = line.partition(":") ++ fields[key.strip()] = value.strip() ++ try: ++ name = fields.get("Name", "?") ++ ppid = int(fields["PPid"]) ++ uid = int(fields["Uid"].split()[0]) ++ state = fields.get("State", "R").split()[0] ++ except (KeyError, ValueError, IndexError): ++ return None ++ return name, ppid, uid, state ++ ++ ++ def snapshot(proc): ++ procs = {} ++ try: ++ entries = os.listdir(proc) ++ except OSError as e: ++ sys.exit(f"::error::runner-quarantine: cannot list {proc}: {e}") ++ for entry in entries: ++ if not entry.isdigit(): ++ continue ++ st = read_status(proc, entry) ++ if st is not None: ++ procs[int(entry)] = st ++ return procs ++ ++ ++ def ancestry(procs, self_pid): ++ chain = set() ++ pid = self_pid ++ while pid > 0 and pid not in chain: ++ chain.add(pid) ++ st = procs.get(pid) ++ if st is None: ++ break ++ pid = st[1] ++ return chain ++ ++ ++ def strays(proc, self_pid, uid): ++ procs = snapshot(proc) ++ keep = ancestry(procs, self_pid) | {1} ++ return { ++ pid: st for pid, st in procs.items() ++ if st[2] == uid and pid not in keep and st[3] not in ("Z", "X") ++ } ++ ++ ++ def signal_all(pids, sig): ++ for pid in pids: ++ try: ++ os.kill(pid, sig) ++ except ProcessLookupError: ++ pass ++ except PermissionError as e: ++ print(f"::warning::runner-quarantine: cannot signal pid {pid}: {e}") ++ ++ ++ def job_env(): ++ """A private HOME for the rest of the job; returns its path.""" ++ temp = os.environ.get("RUNNER_TEMP", "") ++ env_file = os.environ.get("GITHUB_ENV", "") ++ if not temp or not env_file: ++ sys.exit("::error::runner-quarantine: --job-env needs RUNNER_TEMP and GITHUB_ENV") ++ home = tempfile.mkdtemp(prefix="job-home.", dir=temp) # 0700 ++ for sub in (".config", ".cache"): ++ os.mkdir(os.path.join(home, sub), 0o700) ++ gitconfig = os.path.join(home, ".gitconfig") ++ with open(gitconfig, "x"): ++ pass ++ values = { ++ "HOME": home, ++ "XDG_CONFIG_HOME": os.path.join(home, ".config"), ++ "XDG_CACHE_HOME": os.path.join(home, ".cache"), ++ "GIT_CONFIG_GLOBAL": gitconfig, ++ "GIT_CONFIG_NOSYSTEM": "1", ++ } ++ with open(env_file, "a") as f: ++ for key, value in values.items(): ++ f.write(f"{key}={value}\n") ++ return home ++ ++ ++ def main(): ++ ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ++ ap.add_argument("--proc", default="/proc", help="procfs root (a fixture tree in tests)") ++ ap.add_argument("--self", type=int, default=os.getpid(), dest="self_pid", ++ help="the pid whose ancestry is this job (default: this process)") ++ ap.add_argument("--uid", type=int, default=os.getuid(), help="the job's uid (default: this process's)") ++ ap.add_argument("--dry-run", action="store_true", help="name the strays, signal nothing") ++ ap.add_argument("--job-env", action="store_true", ++ help="after a clean quarantine, export a private HOME and git config via GITHUB_ENV") ++ args = ap.parse_args() ++ ++ if args.proc == "/proc": ++ on_runner = os.environ.get("GITHUB_ACTIONS") == "true" ++ server = os.environ.get("GITHUB_SERVER_URL", "") ++ if not on_runner or server in ("", "https://github.com"): ++ print("runner-quarantine: refusing — only a self-hosted Gitea/Forgejo Actions job may end " ++ "every other process of its uid (GITHUB_ACTIONS=true and a non-github.com " ++ "GITHUB_SERVER_URL); use --proc for a fixture.", file=sys.stderr) ++ return 2 ++ ++ rc = quarantine(args) ++ if rc == 0 and args.job_env: ++ print(f"runner-quarantine: private HOME {job_env()}; git reads no global or system config") ++ return rc ++ ++ ++ def quarantine(args): ++ found = strays(args.proc, args.self_pid, args.uid) ++ if not found: ++ print(f"runner-quarantine: no process of uid {args.uid} outside this job's ancestry") ++ return 0 ++ for pid, (name, ppid, _, _) in sorted(found.items()): ++ print(f"::warning::runner-quarantine: pid {pid} ({name}, parent {ppid}) of uid {args.uid} " ++ f"is not part of this job — a process left behind by an earlier job") ++ if args.dry_run: ++ print(f"runner-quarantine: dry run — {len(found)} stray process(es), none signalled") ++ return 0 ++ ++ ended = set() ++ for _ in range(ROUNDS): ++ pids = sorted(found) ++ ended.update(pids) ++ signal_all(pids, signal.SIGSTOP) ++ signal_all(pids, signal.SIGKILL) ++ time.sleep(0.2) ++ found = strays(args.proc, args.self_pid, args.uid) ++ if not found: ++ print(f"runner-quarantine: ended {len(ended)} stray process(es); none left") ++ return 0 ++ names = ", ".join(f"{pid} ({st[0]})" for pid, st in sorted(found.items())) ++ print(f"::error::runner-quarantine: processes of uid {args.uid} survive {ROUNDS} rounds of SIGKILL: " ++ f"{names}. This job holds a signing credential and does not run beside them.") ++ return 1 ++ ++ ++ if __name__ == "__main__": ++ sys.exit(main()) ++ QUARANTINE ++ shell: bash ++ - if: ${{ startsWith(github.ref, 'refs/tags/v') && vars.ZEGIT_SIGNING_RUNNER != '' }} ++ name: Warn when the signing-runner fallback is set (zegit#137) ++ run: echo "::warning::vars.ZEGIT_SIGNING_RUNNER is set to '${FALLBACK}', so this job runs there instead of on the isolated zegit-signing runner. That re-opens zegit#137 until the variable is deleted (documentation/CI-ON-GITEA.md section 11)." ++ env: ++ FALLBACK: ${{ vars.ZEGIT_SIGNING_RUNNER }} ++ continue-on-error: true ++ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 ++ with: ++ fetch-depth: 0 ++ persist-credentials: false ++ - name: Configure private modules from the Gitea mirror (job-scoped) ++ uses: ./.github/actions/gitea-module-auth ++ with: ++ token: ${{ secrets.ZEGIT_CI_READ_TOKEN }} ++ - name: Go toolchain and caches of this job only (sha256-pinned) ++ uses: ./.github/actions/isolated-go ++ - name: go mod download + verify, then vendor (the Dockerfiles build from vendor/) ++ run: | ++ set -euo pipefail ++ go version ++ go mod download ++ go mod verify ++ go mod vendor ++ - name: Private work directory (zegit#138) ++ run: | ++ set -euo pipefail ++ work=$(mktemp -d "${RUNNER_TEMP}/image.XXXXXX") ++ chmod 700 "$work" ++ echo "IMAGE_WORK=$work" >> "$GITHUB_ENV" ++ - name: Install buildctl (workspace-local; runner is non-root) ++ uses: ./.github/actions/fetch-tool ++ with: ++ name: buildctl ++ extract: bin/buildctl ++ url-amd64: https://github.com/moby/buildkit/releases/download/${{ env.BUILDKIT_VERSION }}/buildkit-${{ env.BUILDKIT_VERSION }}.linux-amd64.tar.gz ++ url-arm64: https://github.com/moby/buildkit/releases/download/${{ env.BUILDKIT_VERSION }}/buildkit-${{ env.BUILDKIT_VERSION }}.linux-arm64.tar.gz ++ sha256-amd64: 2975d0f651ad96ba8b80b9992ae1f9a964f4408569af5b6dc36544165c3926af ++ sha256-arm64: 9e8f46bf309ec0ab262967be5538a4dbe06be756a82621f98253933bac5dcf92 ++ - name: buildkitd reachable (mTLS via the sidecar; deny-by-default otherwise) ++ run: | ++ buildctl --version ++ buildctl debug workers ++ - id: meta ++ name: Determine version and build time ++ run: | ++ if [[ "$REF_TYPE" == "tag" ]]; then version="$REF_NAME"; else version="$GITHUB_SHA"; fi ++ echo "version=$version" >> "$GITHUB_OUTPUT" ++ echo "build_time=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT" ++ env: ++ REF_TYPE: ${{ github.ref_type }} ++ REF_NAME: ${{ github.ref_name }} ++ - if: ${{ matrix.image == 'z-backend' }} ++ name: Verify and bake the commit's AoV into the image ++ run: bash scripts/bake-aov.sh "$SHA" cmd/z-backend/buildaov/aov.json ++ env: ++ SHA: ${{ github.sha }} ++ # zegit#185: revocations from the Control Plane, and the root ++ # pinned outside the tree (an owner-set repository variable). ++ CP_URL: https://zegit.io ++ ZEGIT_AOV_ROOT_SHA256: ${{ vars.ZEGIT_AOV_ROOT_SHA256 }} ++ - name: Build (multi-arch) to an OCI archive ++ run: | ++ set -euo pipefail ++ # BuildKit's OCI-layout exporter writes INTO an existing directory, ++ # and run 4909 (2026-09-03) scanned blobs an earlier build left in a ++ # shared /tmp path: the log showed openssh 10.3_p1-r1 being installed ++ # while trivy reported r0 from the stale layout. The layout now lives ++ # in this job's own IMAGE_WORK, and the index is still proven to be ++ # the one this build just wrote. ++ oci="${IMAGE_WORK}/oci" ++ stamp=$(mktemp "${IMAGE_WORK}/stamp.XXXXXX") ++ buildctl build --progress=plain \ ++ --frontend dockerfile.v0 \ ++ --local context=. \ ++ --local dockerfile="cmd/${IMAGE}" \ ++ --opt platform=linux/amd64,linux/arm64 \ ++ --opt build-arg:VERSION="${VERSION}" \ ++ --opt build-arg:COMMIT="${{ github.sha }}" \ ++ --opt build-arg:BUILD_TIME="${BUILD_TIME}" \ ++ --output "type=oci,tar=false,dest=${oci},name=${REGISTRY}/${IMAGE_PATH}/${IMAGE}:${{ github.sha }}-gitea" ++ if [[ ! "${oci}/index.json" -nt "$stamp" ]]; then ++ echo "::error::${oci}/index.json was not written by this build — refusing to scan a stale layout"; exit 1 ++ fi ++ du -sh "$oci"; head -c 400 "${oci}/index.json"; echo ++ env: ++ IMAGE: ${{ matrix.image }} ++ VERSION: ${{ steps.meta.outputs.version }} ++ BUILD_TIME: ${{ steps.meta.outputs.build_time }} ++ - name: Install trivy (workspace-local) ++ uses: ./.github/actions/fetch-tool ++ with: ++ name: trivy ++ extract: trivy ++ url-amd64: https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-64bit.tar.gz ++ url-arm64: https://github.com/aquasecurity/trivy/releases/download/v0.74.0/trivy_0.74.0_Linux-ARM64.tar.gz ++ sha256-amd64: 2ae6fe3ee734b7fdf11335663e18c75ea12dccc76062f09f164a3b0f8be4371a ++ sha256-arm64: b94ce1976bbf3c15b514b605ee88be7c6d94a29be2302847ff01cb794d47aad5 ++ - name: Scan the archive (trivy, fail on fixable HIGH/CRITICAL) ++ run: | ++ # An OCI layout DIRECTORY (buildkit type=oci,tar=false); the multi-arch ++ # index needs an explicit platform. The tarball form failed to ++ # initialise on the first run (2026-08-29). ++ trivy --version ++ # The vulnerability DB is the one network fetch left in a tag build. ++ # Run 5332 (v0.0.147, 2026-09-04) died on it — mirror.gcr.io answered ++ # BLOB_UNKNOWN and ghcr.io DENIED for the same minute — and the agent ++ # token cannot re-run a job, so a transient refusal cost an operator ++ # click. Fetch it once before scanning. Each source is tried on its ++ # own (trivy's comma chain stops at the first DENIED, #76) and the ++ # whole list is retried in rounds with a pause between them; the loop ++ # lives in scripts/trivy-db-download.sh, shared with the rescan. ++ bash scripts/trivy-db-download.sh ++ trivy image --input "${IMAGE_WORK}/oci" --platform linux/amd64 --severity HIGH,CRITICAL --ignore-unfixed --exit-code 1 --format table --scanners vuln --skip-db-update ++ trivy image --input "${IMAGE_WORK}/oci" --platform linux/amd64 --severity MEDIUM,HIGH,CRITICAL --exit-code 0 --format table --scanners vuln --skip-db-update ++ env: ++ IMAGE: ${{ matrix.image }} ++ - name: Install crane (workspace-local) ++ uses: ./.github/actions/fetch-tool ++ with: ++ name: crane ++ extract: crane ++ url-amd64: https://github.com/google/go-containerregistry/releases/download/v0.21.7/go-containerregistry_Linux_x86_64.tar.gz ++ url-arm64: https://github.com/google/go-containerregistry/releases/download/v0.21.7/go-containerregistry_Linux_arm64.tar.gz ++ sha256-amd64: 1a57bc98207fa1c0d04bf760699099e26f8383499bfd55b99c1b919a928a7230 ++ sha256-arm64: b6ee979d9411dfb05ce35ab9e156fe5de7def11a230764a7856ffa2eb971fa88 ++ - id: push ++ name: Push the scanned layout to the Gitea registry (when a package-write token is present) ++ run: | ++ set -euo pipefail ++ if [[ -z "${TOKEN}" ]]; then ++ if [[ "${RELEASE}" == "true" ]]; then ++ echo "::error::ZEGIT_CI_PACKAGE_TOKEN is not set, so this release tag built an image and pushed nothing. A v* run must produce the artefact it claims. Issue a package-write token and store it as this secret." ++ exit 1 ++ fi ++ echo "no ZEGIT_CI_PACKAGE_TOKEN — build-only run (not pushed)"; exit 0 ++ fi ++ mkdir -p "$HOME/.docker" ++ printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY" "$(printf '%s:%s' "$REGISTRY_USER" "$TOKEN" | base64 -w0)" > "$HOME/.docker/config.json" ++ ref="${REGISTRY}/${IMAGE_PATH}/${IMAGE}:${{ github.sha }}-gitea" ++ # The layout's top-level index.json names exactly one manifest: the ++ # multi-arch image index buildkit wrote. That digest is what trivy ++ # scanned; it must be what the registry ends up holding. crane ++ # pushes index.json itself as the tagged manifest, which would nest ++ # an index inside an index — Gitea's registry answers MANIFEST_INVALID ++ # to that (run 8203, 2026-09-17) — so promote the image index blob to ++ # index.json first: the bytes crane then pushes ARE that blob, and ++ # the digest the registry computes is the one trivy scanned. ++ oci="${IMAGE_WORK}/oci" ++ scanned=$(python3 -I -c 'import json,sys; ms=json.load(open(sys.argv[1]))["manifests"]; assert len(ms)==1, ms; print(ms[0]["digest"])' "${oci}/index.json") ++ cp "${oci}/blobs/sha256/${scanned#sha256:}" "${oci}/index.json" ++ [[ "sha256:$(sha256sum "${oci}/index.json" | cut -d' ' -f1)" == "$scanned" ]] || { echo "::error::index.json is not the scanned image index"; exit 1; } ++ crane version ++ crane push --index "$oci" "$ref" ++ pushed=$(crane digest "$ref") ++ if [[ "$pushed" != "$scanned" ]]; then ++ echo "::error::pushed index digest ${pushed} is not the scanned layout's ${scanned} — the registry does not hold the bytes trivy passed. Refusing to sign." ++ exit 1 ++ fi ++ echo "pushed digest: ${pushed} (identical to the scanned layout)" ++ echo "digest=${pushed}" >> "$GITHUB_OUTPUT" ++ env: ++ IMAGE: ${{ matrix.image }} ++ TOKEN: ${{ secrets.ZEGIT_CI_PACKAGE_TOKEN }} ++ RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }} ++ - if: always() ++ name: Remove the scanned OCI layout (host-mode runner; the disk persists) ++ run: | ++ [ -z "${IMAGE_WORK:-}" ] || rm -rf "${IMAGE_WORK}/oci" ++ - name: Install cosign (workspace-local) ++ uses: ./.github/actions/fetch-tool ++ with: ++ name: cosign ++ url-amd64: https://github.com/sigstore/cosign/releases/download/v3.1.3/cosign-linux-amd64 ++ url-arm64: https://github.com/sigstore/cosign/releases/download/v3.1.3/cosign-linux-arm64 ++ sha256-amd64: 4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71 ++ sha256-arm64: c5d324e091826b0d7a78eb16fef316450b4eb9aaec045611c08ba06f5e73220a ++ - name: Install oras (workspace-local) ++ uses: ./.github/actions/fetch-tool ++ with: ++ name: oras ++ extract: oras ++ url-amd64: https://github.com/oras-project/oras/releases/download/v1.3.4/oras_1.3.4_linux_amd64.tar.gz ++ url-arm64: https://github.com/oras-project/oras/releases/download/v1.3.4/oras_1.3.4_linux_arm64.tar.gz ++ sha256-amd64: f27adb935022d94df8dc77719c322dda592c78a0d57a6f7dcdd8d900b248c454 ++ sha256-arm64: 15702c6e3a4a56a8bd8ac5c17efdbcab56d9bada661ccbcf017f5b10c1d89399 ++ - name: Install syft (workspace-local) ++ uses: ./.github/actions/fetch-tool ++ with: ++ name: syft ++ extract: syft ++ url-amd64: https://github.com/anchore/syft/releases/download/v1.51.1/syft_1.51.1_linux_amd64.tar.gz ++ url-arm64: https://github.com/anchore/syft/releases/download/v1.51.1/syft_1.51.1_linux_arm64.tar.gz ++ sha256-amd64: 8fcb33017a0dc1058298c923c436d19dfa68ae93968e0b423248542e3afb9fc3 ++ sha256-arm64: a7fd2b784e6664acd44719270574f6cd8c6864fc2b1700bf9099bd1cccda7d7f ++ - name: Sign with the Control Plane keystore and attach ++ run: | ++ set -euo pipefail ++ if [[ "${RELEASE}" == "true" ]]; then ++ TOKEN="${RELEASE_TOKEN}" SIGN_PATH=/internal/signing/image KEY_PATH=/signing/image-key TOKEN_NAME=ZEGIT_CI_IMAGE_SIGNER_TOKEN PINS="${IMAGE_KEY_PINS}" PIN_FILE=deployment/keys/image-key.pem ++ else ++ TOKEN="${TRIAL_TOKEN}" SIGN_PATH=/internal/signing/image-trial KEY_PATH=/signing/image-trial-key TOKEN_NAME=ZEGIT_CI_IMAGE_TRIAL_SIGNER_TOKEN PINS="${IMAGE_TRIAL_KEY_PINS}" PIN_FILE=deployment/keys/image-trial-key.pem ++ fi ++ if [[ -z "${TOKEN}" || -z "${DIGEST}" ]]; then ++ if [[ "${RELEASE}" == "true" ]]; then ++ # if/else, not `test && assign`: a false `&&` returns non-zero and ++ # `set -e` would kill the step before it printed the error. ++ if [[ -n "${TOKEN}" ]]; then missing="a pushed digest"; else missing="ZEGIT_CI_IMAGE_SIGNER_TOKEN"; fi ++ echo "::error::No ${missing}, so this release image is unsigned. Flux and admission verify the Control Plane's image-signing key, so an unsigned v* image would be refused by the cluster — failing here is the readable failure. Issue a signer-scoped forward token and store it as ZEGIT_CI_IMAGE_SIGNER_TOKEN." ++ exit 1 ++ fi ++ echo "no ${TOKEN_NAME} or no pushed digest — not signed"; exit 0 ++ fi ++ ref="${REGISTRY}/${IMAGE_PATH}/${IMAGE}@${DIGEST}" ++ # The tag rides INSIDE the signed payload (cosign's payload names the ++ # repository only). The Control Plane reads it to require a met Tag ++ # Request quorum for a v* tag before signing; a branch/trial tag ++ # (-gitea) signs without one. The v*-gitea tag itself is written ++ # only after this signature verifies (see the tag step below). ++ # zegit#138: every file of the exchange lives in a fresh 0700 ++ # directory, never a fixed /tmp path another job could swap between ++ # two of these lines, and the payload is checked to name exactly the ++ # pushed digest BEFORE the Control Plane signs it (a signature over ++ # any other digest would spend this tag's one signing slot). ++ # The payload is read ONCE, into a variable: the bytes checked, the ++ # bytes sent and the bytes attached are all decoded from it, and the ++ # attached file is compared with it again right before attaching. ++ sign=$(mktemp -d "${IMAGE_WORK}/sign.XXXXXX") ++ payload_b64=$(cosign generate -a "tag=${VERSION}-gitea" "$ref" | base64 -w0) ++ printf '%s' "$payload_b64" | base64 -d > "$sign/checked.json" ++ bash scripts/signing-payload.sh check "$sign/checked.json" "$DIGEST" ++ # The bearer reaches curl as a config on a pipe, never on its ++ # command line (/proc//cmdline is world-readable; zegit#139). ++ curl -fsS -X POST "${CP_URL}${SIGN_PATH}" -K <(printf 'header = "Authorization: Bearer %s"\n' "$TOKEN") -H 'Content-Type: application/json' \ ++ -d "{\"payload\":\"${payload_b64}\"}" > "$sign/sign.json" ++ python3 -I -c 'import json,sys; print(json.load(open(sys.argv[1]))["signature"])' "$sign/sign.json" > "$sign/sig.b64" ++ # Re-checked after the round trip: still the signed bytes, still the digest. ++ cmp -s "$sign/checked.json" <(printf '%s' "$payload_b64" | base64 -d) \ ++ || { echo "::error::the payload file changed after it was signed — not attaching"; exit 1; } ++ bash scripts/signing-payload.sh check "$sign/checked.json" "$DIGEST" ++ # The same .sig artifact `cosign attach signature` wrote, byte for ++ # byte; cosign v4 removes that command (zegit#39). ++ bash scripts/attach-signature.sh "$ref" "$sign/checked.json" "$sign/sig.b64" ++ curl -fsS "${CP_URL}${KEY_PATH}" > "$sign/image-key.pem" ++ bash scripts/check-key-pin.sh "$sign/image-key.pem" "${PINS}" "${KEY_PATH#/signing/}" "${PIN_FILE}" ++ cosign verify --key "$sign/image-key.pem" --insecure-ignore-tlog "$ref" >/dev/null && echo "verified with the Control Plane's ${KEY_PATH#/signing/} key" ++ env: ++ IMAGE: ${{ matrix.image }} ++ DIGEST: ${{ steps.push.outputs.digest }} ++ RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }} ++ RELEASE_TOKEN: ${{ startsWith(github.ref, 'refs/tags/v') && secrets.ZEGIT_CI_IMAGE_SIGNER_TOKEN || '' }} ++ TRIAL_TOKEN: ${{ secrets.ZEGIT_CI_IMAGE_TRIAL_SIGNER_TOKEN }} ++ VERSION: ${{ steps.meta.outputs.version }} ++ CP_URL: https://zegit.io ++ # zegit#141, #187: the served key must be the one committed under ++ # deployment/keys/ (reviewable in git; scripts/check-key-pin.sh ++ # refuses without a pin). These repository variables are an ++ # OVERRIDE of that pin (fingerprints, comma-separated during a ++ # rotation); leave them unset. ++ IMAGE_KEY_PINS: ${{ vars.ZEGIT_IMAGE_KEY_SHA256 }} ++ IMAGE_TRIAL_KEY_PINS: ${{ vars.ZEGIT_IMAGE_TRIAL_KEY_SHA256 }} ++ - if: ${{ steps.push.outputs.digest != '' }} ++ name: Generate and attach an SPDX SBOM per platform (syft, unsigned referrer) ++ run: | ++ set -euo pipefail ++ syft version ++ bash scripts/image-sbom.sh "${REGISTRY}" "${IMAGE_PATH}/${IMAGE}" "${DIGEST}" "${IMAGE_WORK}/sbom" ++ { ++ echo "### SBOMs — \`${IMAGE}\` @ ${DIGEST}" ++ echo ++ for f in "${IMAGE_WORK}"/sbom/*.spdx.json; do ++ echo "- $(basename "$f"): $(python3 -I -c 'import json,sys; print(len(json.load(open(sys.argv[1])).get("packages", [])))' "$f") packages (attached with oras as cosign's .sbom artifact, unsigned)" ++ done ++ } >> "$GITHUB_STEP_SUMMARY" ++ env: ++ IMAGE: ${{ matrix.image }} ++ DIGEST: ${{ steps.push.outputs.digest }} ++ - if: ${{ startsWith(github.ref, 'refs/tags/v') && steps.push.outputs.digest == '' }} ++ name: Require the SBOM on a release ++ run: | ++ echo "::error::no pushed digest, so no SBOM could be generated or attached for this release image." ++ exit 1 ++ - if: ${{ startsWith(github.ref, 'refs/tags/v') && steps.push.outputs.digest != '' }} ++ name: Tag the signed digest as the release (immutable v*-gitea tag) ++ uses: ./.github/actions/gitea-registry-tag ++ with: ++ registry: ${{ env.REGISTRY }} ++ repository: ${{ env.IMAGE_PATH }}/${{ matrix.image }} ++ digest: ${{ steps.push.outputs.digest }} ++ tag: ${{ github.ref_name }}-gitea ++ user: ${{ env.REGISTRY_USER }} ++ token: ${{ secrets.ZEGIT_CI_PACKAGE_TOKEN }} ++ - if: ${{ startsWith(github.ref, 'refs/tags/v') && steps.push.outputs.digest != '' }} ++ name: Emit the pin bump for this release ++ run: | ++ set -euo pipefail ++ case "$IMAGE" in ++ z-backend) file=deployment/kustomize/base/deployment.yaml ;; ++ z-gateway-server) file=deployment/kustomize/base-gateway/deployment.yaml ;; ++ *) echo "no pin file known for $IMAGE"; exit 0 ;; ++ esac ++ ref="${REGISTRY}/${IMAGE_PATH}/${IMAGE}:${TAG}-gitea@${DIGEST}" ++ { ++ echo "### Pin bump — \`${IMAGE}\` ${TAG}" ++ echo ++ echo "Run in a worktree on master, then commit and \`zg push gateway\`:" ++ echo ++ echo '```sh' ++ # Delimiter is | , not # : the REPLACEMENT ends in " # v0.0.NNN", ++ # so a # delimiter makes sed fail with "unknown option to 's'". ++ # Found by running the emitted command against a real manifest ++ # rather than reading it — a copy-paste that does not run is worse ++ # than none, because it is trusted. ++ # The whole reference, owner included: a release built under a new ++ # registry owner (RUNBOOK-TENANT-PACKAGES.md §3 step 6) moves the ++ # pin to that owner, not just to the new tag and digest. ++ echo "sed -i -E 's|[a-z0-9.-]+(/[a-z0-9._-]+)*/${IMAGE}:v[0-9]+\\.[0-9]+\\.[0-9]+-gitea@sha256:[0-9a-f]{64}( # v[0-9.]+)?|${ref} # ${TAG}|g' \\" ++ echo " ${file}" ++ echo '```' ++ echo ++ echo "Resulting reference:" ++ echo ++ echo '```' ++ echo "$ref" ++ echo '```' ++ } >> "$GITHUB_STEP_SUMMARY" ++ echo "pin bump for $IMAGE written to the job summary" ++ env: ++ IMAGE: ${{ matrix.image }} ++ DIGEST: ${{ steps.push.outputs.digest }} ++ TAG: ${{ github.ref_name }} ++ - if: ${{ always() }} ++ name: Remove job-written credentials from the runner ++ run: | ++ rm -f "$HOME/.docker/config.json" ++ echo "credentials removed" ++ - if: ${{ always() }} ++ name: Sweep module credentials off the runner ++ uses: ./.github/actions/gitea-module-auth ++ with: ++ sweep: "true" ++ - if: ${{ always() }} ++ name: Remove this job's Go toolchain and caches ++ uses: ./.github/actions/isolated-go ++ with: ++ sweep: "true" ++ - if: ${{ always() }} ++ name: Remove this job's work directory ++ run: | ++ [ -z "${IMAGE_WORK:-}" ] || rm -rf "$IMAGE_WORK" ++ timeout-minutes: "30" ++ strategy: ++ fail-fast: "false" ++ matrix: ++ image: ++ - z-gateway-server ++ job-index: 1 ++ job-total: 2 ++ +diff --git a/internal/app/run/workflow_test.go b/internal/app/run/workflow_test.go +index 2c8fafe..d326648 100644 +--- a/internal/app/run/workflow_test.go ++++ b/internal/app/run/workflow_test.go +@@ -4,6 +4,7 @@ + package run + + import ( ++ "os" + "testing" + + runnerv1 "code.forgejo.org/forgejo/actions-proto/runner/v1" +@@ -76,3 +77,21 @@ jobs: + }) + } + } ++ ++// Gitea 28 adds job-index and job-total under strategy in the single-job ++// workflow it sends for a matrix job. testdata/gitea28-matrix-job.yml is the ++// payload of a real task, copied from the error of run 23992. ++func TestGenerateWorkflow_Gitea28MatrixJob(t *testing.T) { ++ payload, err := os.ReadFile("testdata/gitea28-matrix-job.yml") ++ require.NoError(t, err) ++ ++ workflow, jobID, err := generateWorkflow(&runnerv1.Task{WorkflowPayload: payload}) ++ require.NoError(t, err) ++ ++ job := workflow.GetJob(jobID) ++ require.NotNil(t, job) ++ matrices, err := job.GetMatrixes() ++ require.NoError(t, err) ++ require.Len(t, matrices, 1) ++ assert.Equal(t, "z-gateway-server", matrices[0]["image"]) ++}