diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc9e9765d..913e865cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1503,14 +1503,26 @@ jobs: - name: Build Linux image if: matrix.run && matrix.component != 'uhrp-server-cloud-bucket/notifier' run: | - for attempt in 1 2 3; do - if docker build --tag "ts-stack-${{ strategy.job-index }}:ci" "infra/${{ matrix.component }}"; then + # Anonymous base-image pulls can be throttled (HTTP 429). Back off + # longer only for that; any other build failure keeps the short retry. + set -o pipefail + log="$RUNNER_TEMP/docker-build-${{ strategy.job-index }}.log" + for attempt in 1 2 3 4 5; do + if docker build --tag "ts-stack-${{ strategy.job-index }}:ci" "infra/${{ matrix.component }}" 2>&1 | tee "$log"; then exit 0 fi - if [ "$attempt" -lt 3 ]; then - echo "::warning::Docker build attempt $attempt failed; retrying after $((attempt * 5)) seconds." - sleep $((attempt * 5)) + if grep -qiE '429 Too Many Requests|toomanyrequests' "$log"; then + limit=5 + delay=$((15 * (1 << (attempt - 1)))) + else + limit=3 + delay=$((attempt * 5)) + fi + if [ "$attempt" -ge "$limit" ]; then + exit 1 fi + echo "::warning::Docker build attempt $attempt failed; retrying after $delay seconds." + sleep "$delay" done exit 1 diff --git a/scripts/ci-orchestration.test.mjs b/scripts/ci-orchestration.test.mjs index 6a861a5b4..9beaf8cd1 100644 --- a/scripts/ci-orchestration.test.mjs +++ b/scripts/ci-orchestration.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict' import { spawnSync } from 'node:child_process' -import { readFileSync } from 'node:fs' +import { chmodSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' import { join } from 'node:path' import test from 'node:test' @@ -387,3 +388,68 @@ test('the mutation quality job accepts skipped execution only for explicitly emp }) assert.notEqual(failedBuild.status, 0) }) + +// Extract one step's `run: |` block from the raw workflow text, as the other +// checks here read the workflow without a YAML dependency. +function stepScript(workflow, name) { + const lines = workflow.split('\n') + const at = lines.findIndex(line => line.trim() === `- name: ${name}`) + assert.ok(at >= 0, name) + const run = lines.findIndex((line, index) => index > at && line.trim() === 'run: |') + const indent = lines[run + 1].match(/^ */)[0].length + const body = [] + for (const line of lines.slice(run + 1)) { + if (line.trim() !== '' && line.match(/^ */)[0].length < indent) break + body.push(line.slice(indent)) + } + return body.join('\n') +} + +test('infra image builds back off longer only for registry rate limits', () => { + const script = stepScript(readFileSync(CI_PATH, 'utf8'), 'Build Linux image') + .replaceAll('${{ strategy.job-index }}', '0') + .replaceAll('${{ matrix.component }}', 'wab') + const scenario = outcomes => { + const directory = mkdtempSync(join(tmpdir(), 'ts-stack-image-retry-')) + writeFileSync(join(directory, 'outcomes'), outcomes.join('\n') + '\n') + // Each docker call consumes one scripted outcome; sleep only records. + writeFileSync( + join(directory, 'docker'), + `#!/bin/bash\noutcome=$(head -n1 "${directory}/outcomes"); sed -i.bak 1d "${directory}/outcomes"\n` + + `echo call >> "${directory}/calls"\n` + + `case "$outcome" in ok) exit 0;; 429) echo "429 Too Many Requests" >&2; exit 1;; *) echo "error: build failed" >&2; exit 1;; esac\n` + ) + writeFileSync(join(directory, 'sleep'), `#!/bin/bash\necho "$1" >> "${directory}/sleeps"\n`) + chmodSync(join(directory, 'docker'), 0o755) + chmodSync(join(directory, 'sleep'), 0o755) + const run = spawnSync('/bin/bash', ['-e', '-c', script], { + env: { PATH: `${directory}:${process.env.PATH}`, RUNNER_TEMP: directory }, + encoding: 'utf8' + }) + const lines = file => { + try { + return readFileSync(join(directory, file), 'utf8').trim().split('\n').filter(Boolean) + } catch { + return [] + } + } + return { status: run.status, calls: lines('calls').length, sleeps: lines('sleeps').map(Number) } + } + assert.deepEqual(scenario(['ok']), { status: 0, calls: 1, sleeps: [] }) + assert.deepEqual(scenario(['429', '429', '429', 'ok']), { + status: 0, + calls: 4, + sleeps: [15, 30, 60] + }) + assert.deepEqual(scenario(['429', '429', '429', '429', '429']), { + status: 1, + calls: 5, + sleeps: [15, 30, 60, 120] + }) + // A genuine build failure keeps the short retry and does not wait minutes. + assert.deepEqual(scenario(['fail', 'fail', 'fail', 'ok']), { + status: 1, + calls: 3, + sleeps: [5, 10] + }) +})