diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index 623743330..0c67af20d 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -284,21 +284,52 @@ jobs: - name: "Continue functional chain (next: Pool expansion)" # Only chain-triggered runs forward to the next suite; standalone - # workflow_dispatch runs stop after their own cleanup. + # workflow_dispatch runs stop after their own cleanup. A failed + # handoff must never pass silently: it retries, then files an alert + # issue in rustfs/backlog so a stalled chain is visible. if: ${{ always() && github.event_name == 'repository_dispatch' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then + set -uo pipefail + if [ -z "${{GH_TOKEN:-}}" ]; then echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 exit 1 fi - echo "Dispatching next functional suite: Pool expansion" - gh api --method POST repos/rustfs/rustfs/dispatches \ - -f event_type='rustfs-chain-pool' \ - -F 'client_payload[from_suite]=heal' + DISPATCHED=0 + for attempt in 1 2 3; do + if gh api --method POST repos/rustfs/rustfs/dispatches \ + -f event_type='rustfs-chain-pool' \ + -F 'client_payload[from_suite]=heal'; then + echo "dispatched next suite Pool expansion (attempt ${{attempt}})" + DISPATCHED=1 + break + fi + echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2 + sleep "${{attempt}}0" + done + if [ "${{DISPATCHED:-0}}" -ne 1 ]; then + echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2 + TITLE="[functional][chain] stalled after heal (run ${{GITHUB_RUN_ID}})" + BODY_FILE="$(mktemp)" + { + echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts." + echo "" + echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}" + echo "- Expected next event: `rustfs-chain-pool`" + echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." + echo "- Recovery: re-dispatch manually with" + echo " ```" + echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'" + echo " ```" + } > "${{BODY_FILE}}" + gh issue create -R rustfs/backlog --title "${{TITLE}}" \ + --body-file "${{BODY_FILE}}" --label functional-test \ + || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \ + || echo "could not file the stall alert issue either; check the token" >&2 + exit 1 + fi - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml index 1157d2326..2492ca277 100644 --- a/.github/workflows/rustfs-kms-test.yml +++ b/.github/workflows/rustfs-kms-test.yml @@ -340,21 +340,52 @@ jobs: - name: "Continue functional chain (next: Tier)" # Only chain-triggered runs forward to the next suite; standalone - # workflow_dispatch runs stop after their own cleanup. + # workflow_dispatch runs stop after their own cleanup. A failed + # handoff must never pass silently: it retries, then files an alert + # issue in rustfs/backlog so a stalled chain is visible. if: ${{ always() && github.event_name == 'repository_dispatch' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then + set -uo pipefail + if [ -z "${{GH_TOKEN:-}}" ]; then echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 exit 1 fi - echo "Dispatching next functional suite: Tier" - gh api --method POST repos/rustfs/rustfs/dispatches \ - -f event_type='rustfs-chain-tier' \ - -F 'client_payload[from_suite]=kms' + DISPATCHED=0 + for attempt in 1 2 3; do + if gh api --method POST repos/rustfs/rustfs/dispatches \ + -f event_type='rustfs-chain-tier' \ + -F 'client_payload[from_suite]=kms'; then + echo "dispatched next suite Tier (attempt ${{attempt}})" + DISPATCHED=1 + break + fi + echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2 + sleep "${{attempt}}0" + done + if [ "${{DISPATCHED:-0}}" -ne 1 ]; then + echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2 + TITLE="[functional][chain] stalled after kms (run ${{GITHUB_RUN_ID}})" + BODY_FILE="$(mktemp)" + { + echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts." + echo "" + echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}" + echo "- Expected next event: `rustfs-chain-tier`" + echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." + echo "- Recovery: re-dispatch manually with" + echo " ```" + echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'" + echo " ```" + } > "${{BODY_FILE}}" + gh issue create -R rustfs/backlog --title "${{TITLE}}" \ + --body-file "${{BODY_FILE}}" --label functional-test \ + || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \ + || echo "could not file the stall alert issue either; check the token" >&2 + exit 1 + fi - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index 8486293a6..19a49a921 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -596,21 +596,52 @@ jobs: - name: "Continue functional chain (next: Security)" # Only chain-triggered runs forward to the next suite; standalone - # workflow_dispatch runs stop after their own cleanup. + # workflow_dispatch runs stop after their own cleanup. A failed + # handoff must never pass silently: it retries, then files an alert + # issue in rustfs/backlog so a stalled chain is visible. if: ${{ always() && github.event_name == 'repository_dispatch' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then + set -uo pipefail + if [ -z "${{GH_TOKEN:-}}" ]; then echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 exit 1 fi - echo "Dispatching next functional suite: Security" - gh api --method POST repos/rustfs/rustfs/dispatches \ - -f event_type='rustfs-chain-security' \ - -F 'client_payload[from_suite]=pool' + DISPATCHED=0 + for attempt in 1 2 3; do + if gh api --method POST repos/rustfs/rustfs/dispatches \ + -f event_type='rustfs-chain-security' \ + -F 'client_payload[from_suite]=pool'; then + echo "dispatched next suite Security (attempt ${{attempt}})" + DISPATCHED=1 + break + fi + echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2 + sleep "${{attempt}}0" + done + if [ "${{DISPATCHED:-0}}" -ne 1 ]; then + echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2 + TITLE="[functional][chain] stalled after pool (run ${{GITHUB_RUN_ID}})" + BODY_FILE="$(mktemp)" + { + echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts." + echo "" + echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}" + echo "- Expected next event: `rustfs-chain-security`" + echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." + echo "- Recovery: re-dispatch manually with" + echo " ```" + echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'" + echo " ```" + } > "${{BODY_FILE}}" + gh issue create -R rustfs/backlog --title "${{TITLE}}" \ + --body-file "${{BODY_FILE}}" --label functional-test \ + || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \ + || echo "could not file the stall alert issue either; check the token" >&2 + exit 1 + fi - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml index 5757c4992..2666d3496 100644 --- a/.github/workflows/rustfs-s3-compat-test.yml +++ b/.github/workflows/rustfs-s3-compat-test.yml @@ -320,21 +320,52 @@ jobs: - name: "Continue functional chain (next: KMS)" # Only chain-triggered runs forward to the next suite; standalone - # workflow_dispatch runs stop after their own cleanup. + # workflow_dispatch runs stop after their own cleanup. A failed + # handoff must never pass silently: it retries, then files an alert + # issue in rustfs/backlog so a stalled chain is visible. if: ${{ always() && github.event_name == 'repository_dispatch' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then + set -uo pipefail + if [ -z "${{GH_TOKEN:-}}" ]; then echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 exit 1 fi - echo "Dispatching next functional suite: KMS" - gh api --method POST repos/rustfs/rustfs/dispatches \ - -f event_type='rustfs-chain-kms' \ - -F 'client_payload[from_suite]=s3' + DISPATCHED=0 + for attempt in 1 2 3; do + if gh api --method POST repos/rustfs/rustfs/dispatches \ + -f event_type='rustfs-chain-kms' \ + -F 'client_payload[from_suite]=s3'; then + echo "dispatched next suite KMS (attempt ${{attempt}})" + DISPATCHED=1 + break + fi + echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2 + sleep "${{attempt}}0" + done + if [ "${{DISPATCHED:-0}}" -ne 1 ]; then + echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2 + TITLE="[functional][chain] stalled after s3 (run ${{GITHUB_RUN_ID}})" + BODY_FILE="$(mktemp)" + { + echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts." + echo "" + echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}" + echo "- Expected next event: `rustfs-chain-kms`" + echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." + echo "- Recovery: re-dispatch manually with" + echo " ```" + echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'" + echo " ```" + } > "${{BODY_FILE}}" + gh issue create -R rustfs/backlog --title "${{TITLE}}" \ + --body-file "${{BODY_FILE}}" --label functional-test \ + || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \ + || echo "could not file the stall alert issue either; check the token" >&2 + exit 1 + fi - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml index 242d6e831..6a2a86c9d 100644 --- a/.github/workflows/rustfs-storage-test.yml +++ b/.github/workflows/rustfs-storage-test.yml @@ -335,21 +335,52 @@ jobs: - name: "Continue functional chain (next: Heal)" # Only chain-triggered runs forward to the next suite; standalone - # workflow_dispatch runs stop after their own cleanup. + # workflow_dispatch runs stop after their own cleanup. A failed + # handoff must never pass silently: it retries, then files an alert + # issue in rustfs/backlog so a stalled chain is visible. if: ${{ always() && github.event_name == 'repository_dispatch' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then + set -uo pipefail + if [ -z "${{GH_TOKEN:-}}" ]; then echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 exit 1 fi - echo "Dispatching next functional suite: Heal" - gh api --method POST repos/rustfs/rustfs/dispatches \ - -f event_type='rustfs-chain-heal' \ - -F 'client_payload[from_suite]=storage' + DISPATCHED=0 + for attempt in 1 2 3; do + if gh api --method POST repos/rustfs/rustfs/dispatches \ + -f event_type='rustfs-chain-heal' \ + -F 'client_payload[from_suite]=storage'; then + echo "dispatched next suite Heal (attempt ${{attempt}})" + DISPATCHED=1 + break + fi + echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2 + sleep "${{attempt}}0" + done + if [ "${{DISPATCHED:-0}}" -ne 1 ]; then + echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2 + TITLE="[functional][chain] stalled after storage (run ${{GITHUB_RUN_ID}})" + BODY_FILE="$(mktemp)" + { + echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts." + echo "" + echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}" + echo "- Expected next event: `rustfs-chain-heal`" + echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." + echo "- Recovery: re-dispatch manually with" + echo " ```" + echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'" + echo " ```" + } > "${{BODY_FILE}}" + gh issue create -R rustfs/backlog --title "${{TITLE}}" \ + --body-file "${{BODY_FILE}}" --label functional-test \ + || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \ + || echo "could not file the stall alert issue either; check the token" >&2 + exit 1 + fi - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml index 07cf060ad..62dbbc85b 100644 --- a/.github/workflows/rustfs-tier-test.yml +++ b/.github/workflows/rustfs-tier-test.yml @@ -373,21 +373,52 @@ jobs: - name: "Continue functional chain (next: Storage engine)" # Only chain-triggered runs forward to the next suite; standalone - # workflow_dispatch runs stop after their own cleanup. + # workflow_dispatch runs stop after their own cleanup. A failed + # handoff must never pass silently: it retries, then files an alert + # issue in rustfs/backlog so a stalled chain is visible. if: ${{ always() && github.event_name == 'repository_dispatch' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then + set -uo pipefail + if [ -z "${{GH_TOKEN:-}}" ]; then echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 exit 1 fi - echo "Dispatching next functional suite: Storage engine" - gh api --method POST repos/rustfs/rustfs/dispatches \ - -f event_type='rustfs-chain-storage' \ - -F 'client_payload[from_suite]=tier' + DISPATCHED=0 + for attempt in 1 2 3; do + if gh api --method POST repos/rustfs/rustfs/dispatches \ + -f event_type='rustfs-chain-storage' \ + -F 'client_payload[from_suite]=tier'; then + echo "dispatched next suite Storage engine (attempt ${{attempt}})" + DISPATCHED=1 + break + fi + echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2 + sleep "${{attempt}}0" + done + if [ "${{DISPATCHED:-0}}" -ne 1 ]; then + echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2 + TITLE="[functional][chain] stalled after tier (run ${{GITHUB_RUN_ID}})" + BODY_FILE="$(mktemp)" + { + echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts." + echo "" + echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}" + echo "- Expected next event: `rustfs-chain-storage`" + echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." + echo "- Recovery: re-dispatch manually with" + echo " ```" + echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'" + echo " ```" + } > "${{BODY_FILE}}" + gh issue create -R rustfs/backlog --title "${{TITLE}}" \ + --body-file "${{BODY_FILE}}" --label functional-test \ + || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \ + || echo "could not file the stall alert issue either; check the token" >&2 + exit 1 + fi - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml index 60a936c88..ae93f6913 100644 --- a/.github/workflows/rustfs-upgrade-test.yml +++ b/.github/workflows/rustfs-upgrade-test.yml @@ -388,21 +388,52 @@ jobs: - name: "Continue functional chain (next: S3 compatibility)" # Only chain-triggered runs forward to the next suite; standalone - # workflow_dispatch runs stop after their own cleanup. + # workflow_dispatch runs stop after their own cleanup. A failed + # handoff must never pass silently: it retries, then files an alert + # issue in rustfs/backlog so a stalled chain is visible. if: ${{ always() && github.event_name == 'repository_dispatch' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then + set -uo pipefail + if [ -z "${{GH_TOKEN:-}}" ]; then echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2 exit 1 fi - echo "Dispatching next functional suite: S3 compatibility" - gh api --method POST repos/rustfs/rustfs/dispatches \ - -f event_type='rustfs-chain-s3' \ - -F 'client_payload[from_suite]=upgrade' + DISPATCHED=0 + for attempt in 1 2 3; do + if gh api --method POST repos/rustfs/rustfs/dispatches \ + -f event_type='rustfs-chain-s3' \ + -F 'client_payload[from_suite]=upgrade'; then + echo "dispatched next suite S3 compatibility (attempt ${{attempt}})" + DISPATCHED=1 + break + fi + echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2 + sleep "${{attempt}}0" + done + if [ "${{DISPATCHED:-0}}" -ne 1 ]; then + echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2 + TITLE="[functional][chain] stalled after upgrade (run ${{GITHUB_RUN_ID}})" + BODY_FILE="$(mktemp)" + { + echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts." + echo "" + echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}" + echo "- Expected next event: `rustfs-chain-s3`" + echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable." + echo "- Recovery: re-dispatch manually with" + echo " ```" + echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'" + echo " ```" + } > "${{BODY_FILE}}" + gh issue create -R rustfs/backlog --title "${{TITLE}}" \ + --body-file "${{BODY_FILE}}" --label functional-test \ + || gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \ + || echo "could not file the stall alert issue either; check the token" >&2 + exit 1 + fi - name: Notify on failure if: failure()