mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-02 22:27:45 +08:00
ci(functional): retry chain handoffs and alert on stall (#7023)
The repository_dispatch handoff step was continue-on-error with a single attempt: if the call failed (token lacking contents:write, transient API error), the chain stalled silently while every job stayed green. Each handoff now retries 3x and, if all attempts fail, files an alert issue in rustfs/backlog with the exact recovery command before exiting 1 (still continue-on-error, so suite workflows themselves never fail).
This commit is contained in:
hector
45
.github/workflows/rustfs-heal-test.yml
vendored
45
.github/workflows/rustfs-heal-test.yml
vendored
@@ -284,21 +284,52 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: Pool expansion)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
set -uo pipefail
|
||||
if [ -z "${{GH_TOKEN:-}}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Pool expansion"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-pool' \
|
||||
-F 'client_payload[from_suite]=heal'
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-pool' \
|
||||
-F 'client_payload[from_suite]=heal'; then
|
||||
echo "dispatched next suite Pool expansion (attempt ${{attempt}})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
|
||||
sleep "${{attempt}}0"
|
||||
done
|
||||
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Pool expansion after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after heal (run ${{GITHUB_RUN_ID}})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **heal** to **Pool expansion** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
|
||||
echo "- Expected next event: `rustfs-chain-pool`"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
echo " ```"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-pool'"
|
||||
echo " ```"
|
||||
} > "${{BODY_FILE}}"
|
||||
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
|
||||
--body-file "${{BODY_FILE}}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
45
.github/workflows/rustfs-kms-test.yml
vendored
45
.github/workflows/rustfs-kms-test.yml
vendored
@@ -340,21 +340,52 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: Tier)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
set -uo pipefail
|
||||
if [ -z "${{GH_TOKEN:-}}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Tier"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-tier' \
|
||||
-F 'client_payload[from_suite]=kms'
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-tier' \
|
||||
-F 'client_payload[from_suite]=kms'; then
|
||||
echo "dispatched next suite Tier (attempt ${{attempt}})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
|
||||
sleep "${{attempt}}0"
|
||||
done
|
||||
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Tier after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after kms (run ${{GITHUB_RUN_ID}})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **kms** to **Tier** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
|
||||
echo "- Expected next event: `rustfs-chain-tier`"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
echo " ```"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-tier'"
|
||||
echo " ```"
|
||||
} > "${{BODY_FILE}}"
|
||||
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
|
||||
--body-file "${{BODY_FILE}}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
45
.github/workflows/rustfs-pool-expand-test.yml
vendored
45
.github/workflows/rustfs-pool-expand-test.yml
vendored
@@ -596,21 +596,52 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: Security)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
set -uo pipefail
|
||||
if [ -z "${{GH_TOKEN:-}}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Security"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-security' \
|
||||
-F 'client_payload[from_suite]=pool'
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-security' \
|
||||
-F 'client_payload[from_suite]=pool'; then
|
||||
echo "dispatched next suite Security (attempt ${{attempt}})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
|
||||
sleep "${{attempt}}0"
|
||||
done
|
||||
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Security after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after pool (run ${{GITHUB_RUN_ID}})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **pool** to **Security** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
|
||||
echo "- Expected next event: `rustfs-chain-security`"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
echo " ```"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-security'"
|
||||
echo " ```"
|
||||
} > "${{BODY_FILE}}"
|
||||
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
|
||||
--body-file "${{BODY_FILE}}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
45
.github/workflows/rustfs-s3-compat-test.yml
vendored
45
.github/workflows/rustfs-s3-compat-test.yml
vendored
@@ -320,21 +320,52 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: KMS)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
set -uo pipefail
|
||||
if [ -z "${{GH_TOKEN:-}}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: KMS"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-kms' \
|
||||
-F 'client_payload[from_suite]=s3'
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-kms' \
|
||||
-F 'client_payload[from_suite]=s3'; then
|
||||
echo "dispatched next suite KMS (attempt ${{attempt}})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
|
||||
sleep "${{attempt}}0"
|
||||
done
|
||||
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch KMS after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after s3 (run ${{GITHUB_RUN_ID}})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **s3** to **KMS** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
|
||||
echo "- Expected next event: `rustfs-chain-kms`"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
echo " ```"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-kms'"
|
||||
echo " ```"
|
||||
} > "${{BODY_FILE}}"
|
||||
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
|
||||
--body-file "${{BODY_FILE}}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
45
.github/workflows/rustfs-storage-test.yml
vendored
45
.github/workflows/rustfs-storage-test.yml
vendored
@@ -335,21 +335,52 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: Heal)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
set -uo pipefail
|
||||
if [ -z "${{GH_TOKEN:-}}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Heal"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-heal' \
|
||||
-F 'client_payload[from_suite]=storage'
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-heal' \
|
||||
-F 'client_payload[from_suite]=storage'; then
|
||||
echo "dispatched next suite Heal (attempt ${{attempt}})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
|
||||
sleep "${{attempt}}0"
|
||||
done
|
||||
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Heal after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after storage (run ${{GITHUB_RUN_ID}})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **storage** to **Heal** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
|
||||
echo "- Expected next event: `rustfs-chain-heal`"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
echo " ```"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-heal'"
|
||||
echo " ```"
|
||||
} > "${{BODY_FILE}}"
|
||||
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
|
||||
--body-file "${{BODY_FILE}}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
45
.github/workflows/rustfs-tier-test.yml
vendored
45
.github/workflows/rustfs-tier-test.yml
vendored
@@ -373,21 +373,52 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: Storage engine)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
set -uo pipefail
|
||||
if [ -z "${{GH_TOKEN:-}}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: Storage engine"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-storage' \
|
||||
-F 'client_payload[from_suite]=tier'
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-storage' \
|
||||
-F 'client_payload[from_suite]=tier'; then
|
||||
echo "dispatched next suite Storage engine (attempt ${{attempt}})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
|
||||
sleep "${{attempt}}0"
|
||||
done
|
||||
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Storage engine after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after tier (run ${{GITHUB_RUN_ID}})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **tier** to **Storage engine** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
|
||||
echo "- Expected next event: `rustfs-chain-storage`"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
echo " ```"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-storage'"
|
||||
echo " ```"
|
||||
} > "${{BODY_FILE}}"
|
||||
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
|
||||
--body-file "${{BODY_FILE}}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
45
.github/workflows/rustfs-upgrade-test.yml
vendored
45
.github/workflows/rustfs-upgrade-test.yml
vendored
@@ -388,21 +388,52 @@ jobs:
|
||||
|
||||
- name: "Continue functional chain (next: S3 compatibility)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup.
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff must never pass silently: it retries, then files an alert
|
||||
# issue in rustfs/backlog so a stalled chain is visible.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
set -uo pipefail
|
||||
if [ -z "${{GH_TOKEN:-}}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Dispatching next functional suite: S3 compatibility"
|
||||
gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-s3' \
|
||||
-F 'client_payload[from_suite]=upgrade'
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-s3' \
|
||||
-F 'client_payload[from_suite]=upgrade'; then
|
||||
echo "dispatched next suite S3 compatibility (attempt ${{attempt}})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${{attempt}} failed; retrying in ${{attempt}}0s" >&2
|
||||
sleep "${{attempt}}0"
|
||||
done
|
||||
if [ "${{DISPATCHED:-0}}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch S3 compatibility after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after upgrade (run ${{GITHUB_RUN_ID}})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **upgrade** to **S3 compatibility** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${{GITHUB_SERVER_URL}}/${{GITHUB_REPOSITORY}}/actions/runs/${{GITHUB_RUN_ID}}"
|
||||
echo "- Expected next event: `rustfs-chain-s3`"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
echo " ```"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-s3'"
|
||||
echo " ```"
|
||||
} > "${{BODY_FILE}}"
|
||||
gh issue create -R rustfs/backlog --title "${{TITLE}}" \
|
||||
--body-file "${{BODY_FILE}}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${{TITLE}}" --body-file "${{BODY_FILE}}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
Reference in New Issue
Block a user