diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml
new file mode 100644
index 000000000..4b1c8f5d0
--- /dev/null
+++ b/.github/workflows/rustfs-functional-chain.yml
@@ -0,0 +1,74 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Functional chain driver: runs the nine functional suites in a fixed order
+# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security, with
+# performance on its own runner in parallel) and guarantees the chain keeps
+# moving even when individual suites fail.
+#
+# Each suite workflow can still be dispatched standalone (workflow_dispatch);
+# only chain-triggered runs forward to the next suite via repository_dispatch,
+# so a standalone run never drags the rest of the chain behind it.
+#
+# Why not workflow_run chaining: GitHub does not guarantee delivery of
+# workflow_run events (they are fire-and-forget), and the head-SHA filter made
+# newly added suites (storage) unable to trigger at all. Explicit
+# repository_dispatch handoffs are verifiable and re-drivable.
+
+name: RustFS Functional Chain
+
+on:
+ workflow_dispatch:
+ workflow_run:
+ # Entry point: start the chain after the nightly build completes. The
+ # build's own conclusion does not gate the chain; each suite reports its
+ # own result to rustfs/backlog and the dashboard.
+ workflows: ["Nightly GNU Build"]
+ types: [completed]
+
+permissions:
+ contents: read
+
+jobs:
+ start-chain:
+ name: Start functional chain (upgrade first)
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ if: ${{ github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.event == 'schedule') }}
+ steps:
+ - name: Dispatch first suite (upgrade)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; cannot start the functional chain" >&2
+ exit 1
+ fi
+ gh api --method POST repos/rustfs/rustfs/dispatches \
+ -f event_type='rustfs-chain-upgrade' \
+ -F 'client_payload[from_suite]=nightly-build'
+
+ - name: Dispatch performance suite (parallel, own runner)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch performance" >&2
+ exit 1
+ fi
+ gh api --method POST repos/rustfs/rustfs/dispatches \
+ -f event_type='rustfs-chain-performance' \
+ -F 'client_payload[from_suite]=nightly-build'
diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml
index c2c6e6f07..623743330 100644
--- a/.github/workflows/rustfs-heal-test.yml
+++ b/.github/workflows/rustfs-heal-test.yml
@@ -23,6 +23,11 @@ on:
description: 'Reset the nodes after the test (DESTROYS test data/config)'
type: boolean
default: true
+ repository_dispatch:
+ # Chain handoff: dispatched when the storage suite finishes. Heal runs
+ # exactly once per chain; the pool expansion workflow no longer embeds
+ # its own heal pass.
+ types: [rustfs-chain-heal]
permissions:
contents: read
@@ -49,19 +54,33 @@ env:
jobs:
heal-test:
runs-on: smoke-testing
+ # Requirement: a failing suite must not fail the workflow; failures
+ # are filed to rustfs/backlog and the chain continues.
+ continue-on-error: true
timeout-minutes: 480
- # Manual-only standalone run. Nightly chain already runs heal in
- # rustfs-pool-expand-test.yml to avoid duplicate heal executions.
- if: ${{ github.event_name == 'workflow_dispatch' }}
+ # Standalone manual run, or one link of the nightly functional chain
+ # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
+ if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- - name: Checkout auto-testing scripts
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- with:
- repository: rustfs/auto-testing
- ref: main
- path: auto-testing
- persist-credentials: false
- token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ # auto-testing is private: clone it with the dedicated PF token (not
+ # GITHUB_TOKEN) and retry transient GitHub/network failures.
+ - name: Checkout auto-testing scripts (with retry)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ rm -rf auto-testing
+ for attempt in 1 2 3 4 5; do
+ if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+ echo "auto-testing cloned (attempt ${attempt})"
+ exit 0
+ fi
+ rm -rf auto-testing
+ echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+ sleep $((attempt * 15))
+ done
+ echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+ exit 1
- name: Show environment
run: |
@@ -116,8 +135,8 @@ jobs:
./auto-testing/rustfs_heal_test.sh \
--steps "3,4,5,6,7" -y \
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
- --stop-node-gb "${{ inputs.stop_node_gb }}" \
- --warp-stop-gb "${{ inputs.warp_stop_gb }}" \
+ --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
+ --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
--log-file /tmp/rustfs-heal-test.log
- name: Generate report
@@ -175,155 +194,64 @@ jobs:
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
- cat > /tmp/rustfs-functional-index.html <<'EOF'
-
-
-
-
-
- RustFS Functional Test Reports
-
-
-
-
-
-
RustFS Functional Test Reports
-
Select a suite and date to view the build version used in that run.
-
-
-
-
-
-
-
-
- EOF
-
- INDEX_PATH="functional/index.html"
- INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
- INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
- if [ -n "${INDEX_SHA}" ]; then
- jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
- else
- jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
- '{message:$msg, content:$content}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+ - name: File failure issue in rustfs/backlog
+ if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ SUITE: 'heal'
+ SUITE_LABEL: 'Heal'
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ REPORT_FILE: '/tmp/rustfs-heal-report.md'
+ LOG_FILE: '/tmp/rustfs-heal-test.log'
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+ exit 0
fi
+ TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+ EXISTING="$(gh issue list -R rustfs/backlog --state all \
+ --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+ --json number --jq '.[].number' || true)"
+ if [ -n "${EXISTING}" ]; then
+ echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+ exit 0
+ fi
+ redact() {
+ sed -E \
+ -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+ -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+ }
+ BODY_FILE="$(mktemp)"
+ {
+ echo "The **${SUITE_LABEL}** functional suite failed."
+ echo ""
+ echo "- Suite: \`${SUITE}\`"
+ echo "- Run: ${RUN_URL}"
+ echo "- Trigger: ${GITHUB_EVENT_NAME}"
+ echo "- Date: $(date -u +%Y-%m-%d)"
+ echo ""
+ echo "## Report (errors and symptoms)"
+ echo ""
+ if [ -s "${REPORT_FILE}" ]; then
+ redact < "${REPORT_FILE}"
+ elif [ -s "${LOG_FILE:-}" ]; then
+ echo "(report file missing; log tail below)"
+ echo ""
+ tail -n 200 "${LOG_FILE}" | redact
+ else
+ echo "(no report or log file was produced)"
+ fi
+ } | head -c 55000 > "${BODY_FILE}"
+ gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+ if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+ --body-file "${BODY_FILE}" --label functional-test; then
+ gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+ fi
+ echo "filed backlog issue for suite ${SUITE}"
- name: Upload test logs
if: always()
@@ -354,6 +282,24 @@ jobs:
'
done
+ - name: "Continue functional chain (next: Pool expansion)"
+ # Only chain-triggered runs forward to the next suite; standalone
+ # workflow_dispatch runs stop after their own cleanup.
+ if: ${{ always() && github.event_name == 'repository_dispatch' }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+ exit 1
+ fi
+ echo "Dispatching next functional suite: Pool expansion"
+ gh api --method POST repos/rustfs/rustfs/dispatches \
+ -f event_type='rustfs-chain-pool' \
+ -F 'client_payload[from_suite]=heal'
+
- name: Notify on failure
if: failure()
run: |
diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml
index 8f0f55f77..1157d2326 100644
--- a/.github/workflows/rustfs-kms-test.yml
+++ b/.github/workflows/rustfs-kms-test.yml
@@ -23,10 +23,9 @@ on:
description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
required: false
type: string
- workflow_run:
- # Strict shared-environment order: run after S3 compatibility test completes.
- workflows: ["RustFS S3 Compatibility Test"]
- types: [completed]
+ repository_dispatch:
+ # Chain handoff: dispatched when the S3 compatibility suite finishes.
+ types: [rustfs-chain-kms]
permissions:
contents: read
@@ -52,16 +51,27 @@ jobs:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 420
- if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
+ if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- - name: Checkout auto-testing scripts
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- with:
- repository: rustfs/auto-testing
- ref: main
- path: auto-testing
- persist-credentials: false
- token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ # auto-testing is private: clone it with the dedicated PF token (not
+ # GITHUB_TOKEN) and retry transient GitHub/network failures.
+ - name: Checkout auto-testing scripts (with retry)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ rm -rf auto-testing
+ for attempt in 1 2 3 4 5; do
+ if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+ echo "auto-testing cloned (attempt ${attempt})"
+ exit 0
+ fi
+ rm -rf auto-testing
+ echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+ sleep $((attempt * 15))
+ done
+ echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+ exit 1
- name: Show environment
run: |
@@ -240,105 +250,64 @@ jobs:
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
- cat > /tmp/rustfs-functional-index.html <<'EOF'
-
-
-
-
-
- RustFS Functional Test Reports
-
-
-
-
-
-
RustFS Functional Test Reports
-
S3, KMS, Tier report tabs. Each tab lists reports by date.
-
-
-
-
-
-
-
- EOF
-
- INDEX_PATH="functional/index.html"
- INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
- INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
- if [ -n "${INDEX_SHA}" ]; then
- jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
- else
- jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
- '{message:$msg, content:$content}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+ - name: File failure issue in rustfs/backlog
+ if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ SUITE: 'kms'
+ SUITE_LABEL: 'KMS'
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ REPORT_FILE: '/tmp/rustfs-kms-report.md'
+ LOG_FILE: '/tmp/rustfs-kms.log'
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+ exit 0
fi
+ TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+ EXISTING="$(gh issue list -R rustfs/backlog --state all \
+ --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+ --json number --jq '.[].number' || true)"
+ if [ -n "${EXISTING}" ]; then
+ echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+ exit 0
+ fi
+ redact() {
+ sed -E \
+ -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+ -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+ }
+ BODY_FILE="$(mktemp)"
+ {
+ echo "The **${SUITE_LABEL}** functional suite failed."
+ echo ""
+ echo "- Suite: \`${SUITE}\`"
+ echo "- Run: ${RUN_URL}"
+ echo "- Trigger: ${GITHUB_EVENT_NAME}"
+ echo "- Date: $(date -u +%Y-%m-%d)"
+ echo ""
+ echo "## Report (errors and symptoms)"
+ echo ""
+ if [ -s "${REPORT_FILE}" ]; then
+ redact < "${REPORT_FILE}"
+ elif [ -s "${LOG_FILE:-}" ]; then
+ echo "(report file missing; log tail below)"
+ echo ""
+ tail -n 200 "${LOG_FILE}" | redact
+ else
+ echo "(no report or log file was produced)"
+ fi
+ } | head -c 55000 > "${BODY_FILE}"
+ gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+ if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+ --body-file "${BODY_FILE}" --label functional-test; then
+ gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+ fi
+ echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
@@ -369,6 +338,24 @@ jobs:
'
done
+ - name: "Continue functional chain (next: Tier)"
+ # Only chain-triggered runs forward to the next suite; standalone
+ # workflow_dispatch runs stop after their own cleanup.
+ if: ${{ always() && github.event_name == 'repository_dispatch' }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+ exit 1
+ fi
+ echo "Dispatching next functional suite: Tier"
+ gh api --method POST repos/rustfs/rustfs/dispatches \
+ -f event_type='rustfs-chain-tier' \
+ -F 'client_payload[from_suite]=kms'
+
- name: Notify on failure
if: failure()
run: |
diff --git a/.github/workflows/rustfs-performance-test.yml b/.github/workflows/rustfs-performance-test.yml
index ec64f7480..6d960052c 100644
--- a/.github/workflows/rustfs-performance-test.yml
+++ b/.github/workflows/rustfs-performance-test.yml
@@ -48,10 +48,10 @@ on:
description: 'Reset the nodes after the test (DESTROYS test data/config)'
type: boolean
default: true
- workflow_run:
- # Run after the nightly build completes; the nightly deb is what the test installs.
- workflows: ["Nightly GNU Build"]
- types: [completed]
+ repository_dispatch:
+ # Chain entry: dispatched by rustfs-functional-chain.yml (runs on its own
+ # pf-testing runner, in parallel with the shared-VM chain).
+ types: [rustfs-chain-performance]
permissions:
contents: read
@@ -84,19 +84,33 @@ env:
jobs:
performance-test:
runs-on: pf-testing
+ # Requirement: a failing benchmark must not fail the workflow;
+ # failures are filed to rustfs/backlog.
+ continue-on-error: true
timeout-minutes: 900
# Run on manual dispatch, or when the nightly build completed successfully.
# Skipped when nightly failed.
- if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+ if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- - name: Checkout auto-testing scripts
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- with:
- repository: rustfs/auto-testing
- ref: main
- path: auto-testing
- persist-credentials: false
- token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ # auto-testing is private: clone it with the dedicated PF token (not
+ # GITHUB_TOKEN) and retry transient GitHub/network failures.
+ - name: Checkout auto-testing scripts (with retry)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ rm -rf auto-testing
+ for attempt in 1 2 3 4 5; do
+ if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+ echo "auto-testing cloned (attempt ${attempt})"
+ exit 0
+ fi
+ rm -rf auto-testing
+ echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+ sleep $((attempt * 15))
+ done
+ echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+ exit 1
- name: Show environment
run: |
@@ -212,6 +226,65 @@ jobs:
echo "created ${REPORT_PATH} in rustfs/dashboard"
fi
+ - name: File failure issue in rustfs/backlog
+ if: ${{ always() && (failure() || steps.benchmark.outcome == 'failure' || steps.benchmark.outcome == 'cancelled') }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ SUITE: 'performance'
+ SUITE_LABEL: 'Performance'
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ REPORT_FILE: '/tmp/rustfs-perf-report.md'
+ LOG_FILE: '/tmp/rustfs-perf-test.log'
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+ exit 0
+ fi
+ TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+ EXISTING="$(gh issue list -R rustfs/backlog --state all \
+ --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+ --json number --jq '.[].number' || true)"
+ if [ -n "${EXISTING}" ]; then
+ echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+ exit 0
+ fi
+ redact() {
+ sed -E \
+ -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+ -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+ }
+ BODY_FILE="$(mktemp)"
+ {
+ echo "The **${SUITE_LABEL}** functional suite failed."
+ echo ""
+ echo "- Suite: \`${SUITE}\`"
+ echo "- Run: ${RUN_URL}"
+ echo "- Trigger: ${GITHUB_EVENT_NAME}"
+ echo "- Date: $(date -u +%Y-%m-%d)"
+ echo ""
+ echo "## Report (errors and symptoms)"
+ echo ""
+ if [ -s "${REPORT_FILE}" ]; then
+ redact < "${REPORT_FILE}"
+ elif [ -s "${LOG_FILE:-}" ]; then
+ echo "(report file missing; log tail below)"
+ echo ""
+ tail -n 200 "${LOG_FILE}" | redact
+ else
+ echo "(no report or log file was produced)"
+ fi
+ } | head -c 55000 > "${BODY_FILE}"
+ gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+ if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+ --body-file "${BODY_FILE}" --label functional-test; then
+ gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+ fi
+ echo "filed backlog issue for suite ${SUITE}"
+
- name: Upload test logs & results
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml
index 8ab3f17b4..8486293a6 100644
--- a/.github/workflows/rustfs-pool-expand-test.yml
+++ b/.github/workflows/rustfs-pool-expand-test.yml
@@ -1,4 +1,4 @@
-name: RustFS Pool Expansion / Heal Test
+name: RustFS Pool Expansion Test
on:
workflow_dispatch:
@@ -33,14 +33,6 @@ on:
description: 'Run the pool decommission step (3-pool topology only)'
type: boolean
default: true
- stop_node_gb:
- description: 'Heal: stop the outage node when surviving nodes reach N GiB'
- required: false
- default: '15'
- warp_stop_gb:
- description: 'Heal: stop warp when surviving nodes reach N GiB'
- required: false
- default: '40'
cleanup_before:
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
type: boolean
@@ -49,17 +41,16 @@ on:
description: 'Reset the nodes after the test (DESTROYS test data/config)'
type: boolean
default: true
- workflow_run:
- # Strict shared-environment order: run after tier test completes.
- workflows: ["RustFS Tier Test"]
- types: [completed]
+ repository_dispatch:
+ # Chain handoff: dispatched when the heal suite finishes.
+ types: [rustfs-chain-pool]
permissions:
contents: read
-# Only one test run at a time: every job mutates the same shared test
+# Only one test run at a time: the job mutates the same shared test
# environment (vm000/vm001/vm002), so concurrent runs must not clobber each
-# other. Jobs inside a run are chained with needs to serialize them.
+# other.
concurrency:
group: rustfs-shared-functional-tests
cancel-in-progress: false
@@ -80,330 +71,16 @@ env:
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
jobs:
- heal-test:
- name: Heal test
- runs-on: smoke-testing
- timeout-minutes: 480
- if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
- steps:
- - name: Checkout auto-testing scripts
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- with:
- repository: rustfs/auto-testing
- ref: main
- path: auto-testing
- persist-credentials: false
- token: ${{ secrets.PF_TESTING_GH_TOKEN }}
-
- - name: Show environment
- run: |
- uname -a
- jq --version
- openssl version
- df -h /data | tail -1
-
- - name: Cleanup environment (before)
- if: ${{ inputs.cleanup_before != 'false' }}
- run: |
- set -euo pipefail
- read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
- SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
- for node in "${NODES[@]}"; do
- ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
- set -euo pipefail
- SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
- ${SUDO} systemctl stop rustfs 2>/dev/null || true
- if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
- ${SUDO} dpkg -P rustfs
- fi
- for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
- ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
- '
- done
-
- - name: Install RustFS package & start cluster
- run: |
- ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
- if [ -n "${{ inputs.package_url }}" ]; then
- ARGS+=(--package-url "${{ inputs.package_url }}")
- else
- ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
- fi
- ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
- - name: Preflight checks
- run: |
- ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}")
- if [ -n "${{ inputs.package_url }}" ]; then
- ARGS+=(--package-url "${{ inputs.package_url }}")
- else
- ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
- fi
- ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
- - name: Run heal test (write -> outage -> heal -> verify)
- id: test
- run: |
- ARGS=(--steps "3,4,5,6,7" -y \
- --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
- --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \
- --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \
- --log-file /tmp/rustfs-heal-test.log)
- if [ -n "${{ inputs.package_url }}" ]; then
- ARGS+=(--package-url "${{ inputs.package_url }}")
- else
- ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
- fi
- ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
-
- - name: Generate report
- if: always()
- env:
- LOG_FILE: /tmp/rustfs-heal-test.log
- REPORT_FILE: /tmp/rustfs-heal-report.md
- run: |
- set -euo pipefail
- PACKAGE_URL='${{ inputs.package_url }}'
- if [ -n "${PACKAGE_URL}" ]; then
- PACKAGE_SOURCE="${PACKAGE_URL}"
- else
- PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
- fi
- {
- echo "# RustFS heal test report"
- echo ""
- echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
- echo "- Trigger: ${{ github.event_name }}"
- echo "- Package: ${PACKAGE_SOURCE}"
- echo "- Test Step Outcome: ${{ steps.test.outcome }}"
- echo ""
- echo "## Log tail"
- echo '```text'
- tail -n 200 "${LOG_FILE}" || true
- echo '```'
- } | tee "${REPORT_FILE}"
- cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-
- - name: Upload functional report to dashboard
- if: always()
- continue-on-error: true
- env:
- GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
- REPORT_FILE: /tmp/rustfs-heal-report.md
- SUITE: heal
- run: |
- set -euo pipefail
- if [ -z "${GH_TOKEN:-}" ]; then
- echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
- exit 0
- fi
- DATE="$(date -u +%Y-%m-%d)"
- REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
- CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
- SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
- if [ -n "${SHA}" ]; then
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
- else
- jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
- '{message:$msg, content:$content}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
- fi
-
- cat > /tmp/rustfs-functional-index.html <<'EOF'
-
-
-
-
-
- RustFS Functional Test Reports
-
-
-
-
-
-
RustFS Functional Test Reports
-
Select a suite and date to view the build version used in that run.
-
-
-
-
-
-
-
-
- EOF
-
- INDEX_PATH="functional/index.html"
- INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
- INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
- if [ -n "${INDEX_SHA}" ]; then
- jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
- else
- jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
- '{message:$msg, content:$content}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
- fi
-
- - name: Upload test logs
- if: always()
- uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
- with:
- name: rustfs-heal-test-${{ github.run_id }}
- path: |
- /tmp/rustfs-heal-test.log
- /tmp/rustfs-warp.*.log
- if-no-files-found: warn
-
- - name: Cleanup environment (after)
- if: ${{ always() && inputs.cleanup_after != 'false' }}
- run: |
- set -euo pipefail
- read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
- SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
- for node in "${NODES[@]}"; do
- ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
- set -euo pipefail
- SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
- ${SUDO} systemctl stop rustfs 2>/dev/null || true
- if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
- ${SUDO} dpkg -P rustfs
- fi
- for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
- ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
- '
- done
-
- - name: Notify on failure
- if: failure()
- run: |
- echo "RustFS heal test failed"
- echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
- echo "See the uploaded log artifact for details."
-
- # Pool expansion runs after heal regardless of heal outcome.
+ # Pool expansion: dispatched by the heal suite's chain handoff. Heal
+ # itself lives in rustfs-heal-test.yml and runs exactly once per chain.
pool-expansion-test:
name: Pool expansion / decommission test
runs-on: smoke-testing
+ # Requirement: a failing suite must not fail the workflow; failures
+ # are filed to rustfs/backlog and the chain continues.
+ continue-on-error: true
timeout-minutes: 360
- needs: heal-test
- if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run') }}
+ if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
env:
RUSTFS_POOL_ADMIN_ENDPOINT: ${{ secrets.RUSTFS_POOL_ADMIN_ENDPOINT || vars.RUSTFS_POOL_ADMIN_ENDPOINT || 'http://rustfs-node1:9000' }}
RUSTFS_POOL_PROXY_ENDPOINT: http://127.0.0.1:19000
@@ -411,14 +88,25 @@ jobs:
RUSTFS_SHARED_PROXY_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
RUSTFS_POOL_NODE_ENDPOINTS: ${{ secrets.RUSTFS_POOL_NODE_ENDPOINTS || vars.RUSTFS_POOL_NODE_ENDPOINTS || 'http://rustfs-node1:9000 http://rustfs-node2:9000 http://rustfs-node3:9000' }}
steps:
- - name: Checkout auto-testing scripts
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- with:
- repository: rustfs/auto-testing
- ref: main
- path: auto-testing
- persist-credentials: false
- token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ # auto-testing is private: clone it with the dedicated PF token (not
+ # GITHUB_TOKEN) and retry transient GitHub/network failures.
+ - name: Checkout auto-testing scripts (with retry)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ rm -rf auto-testing
+ for attempt in 1 2 3 4 5; do
+ if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+ echo "auto-testing cloned (attempt ${attempt})"
+ exit 0
+ fi
+ rm -rf auto-testing
+ echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+ sleep $((attempt * 15))
+ done
+ echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+ exit 1
- name: Initialize pool test artifacts
run: |
@@ -810,155 +498,64 @@ jobs:
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
- cat > /tmp/rustfs-functional-index.html <<'EOF'
-
-
-
-
-
- RustFS Functional Test Reports
-
-
-
-
-
-
RustFS Functional Test Reports
-
Select a suite and date to view the build version used in that run.
-
-
-
-
-
-
-
-
- EOF
-
- INDEX_PATH="functional/index.html"
- INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
- INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
- if [ -n "${INDEX_SHA}" ]; then
- jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
- else
- jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
- '{message:$msg, content:$content}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+ - name: File failure issue in rustfs/backlog
+ if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ SUITE: 'pool'
+ SUITE_LABEL: 'Pool expansion'
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ REPORT_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-report.md'
+ LOG_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-test.log'
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+ exit 0
fi
+ TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+ EXISTING="$(gh issue list -R rustfs/backlog --state all \
+ --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+ --json number --jq '.[].number' || true)"
+ if [ -n "${EXISTING}" ]; then
+ echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+ exit 0
+ fi
+ redact() {
+ sed -E \
+ -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+ -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+ }
+ BODY_FILE="$(mktemp)"
+ {
+ echo "The **${SUITE_LABEL}** functional suite failed."
+ echo ""
+ echo "- Suite: \`${SUITE}\`"
+ echo "- Run: ${RUN_URL}"
+ echo "- Trigger: ${GITHUB_EVENT_NAME}"
+ echo "- Date: $(date -u +%Y-%m-%d)"
+ echo ""
+ echo "## Report (errors and symptoms)"
+ echo ""
+ if [ -s "${REPORT_FILE}" ]; then
+ redact < "${REPORT_FILE}"
+ elif [ -s "${LOG_FILE:-}" ]; then
+ echo "(report file missing; log tail below)"
+ echo ""
+ tail -n 200 "${LOG_FILE}" | redact
+ else
+ echo "(no report or log file was produced)"
+ fi
+ } | head -c 55000 > "${BODY_FILE}"
+ gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+ if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+ --body-file "${BODY_FILE}" --label functional-test; then
+ gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+ fi
+ echo "filed backlog issue for suite ${SUITE}"
- name: Upload test logs
if: always()
@@ -997,6 +594,24 @@ jobs:
'
done
+ - name: "Continue functional chain (next: Security)"
+ # Only chain-triggered runs forward to the next suite; standalone
+ # workflow_dispatch runs stop after their own cleanup.
+ if: ${{ always() && github.event_name == 'repository_dispatch' }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+ exit 1
+ fi
+ echo "Dispatching next functional suite: Security"
+ gh api --method POST repos/rustfs/rustfs/dispatches \
+ -f event_type='rustfs-chain-security' \
+ -F 'client_payload[from_suite]=pool'
+
- name: Notify on failure
if: failure()
run: |
diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml
index fd25ae601..5757c4992 100644
--- a/.github/workflows/rustfs-s3-compat-test.yml
+++ b/.github/workflows/rustfs-s3-compat-test.yml
@@ -11,10 +11,9 @@ on:
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
required: false
type: string
- workflow_run:
- # Run after upgrade compatibility completes; the nightly deb is what the test installs.
- workflows: ["RustFS Upgrade Test"]
- types: [completed]
+ repository_dispatch:
+ # Chain handoff: dispatched when the upgrade suite finishes.
+ types: [rustfs-chain-s3]
permissions:
contents: read
@@ -40,16 +39,27 @@ jobs:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360
- if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+ if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- - name: Checkout auto-testing scripts
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- with:
- repository: rustfs/auto-testing
- ref: main
- path: auto-testing
- persist-credentials: false
- token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ # auto-testing is private: clone it with the dedicated PF token (not
+ # GITHUB_TOKEN) and retry transient GitHub/network failures.
+ - name: Checkout auto-testing scripts (with retry)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ rm -rf auto-testing
+ for attempt in 1 2 3 4 5; do
+ if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+ echo "auto-testing cloned (attempt ${attempt})"
+ exit 0
+ fi
+ rm -rf auto-testing
+ echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+ sleep $((attempt * 15))
+ done
+ echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+ exit 1
- name: Show environment
run: |
@@ -220,155 +230,64 @@ jobs:
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
- cat > /tmp/rustfs-functional-index.html <<'EOF'
-
-
-
-
-
- RustFS Functional Test Reports
-
-
-
-
-
-
RustFS Functional Test Reports
-
Select a suite and date to view the build version used in that run.
-
-
-
-
-
-
-
-
- EOF
-
- INDEX_PATH="functional/index.html"
- INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
- INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
- if [ -n "${INDEX_SHA}" ]; then
- jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
- else
- jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
- '{message:$msg, content:$content}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+ - name: File failure issue in rustfs/backlog
+ if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ SUITE: 's3'
+ SUITE_LABEL: 'S3 compatibility'
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ REPORT_FILE: '/tmp/rustfs-s3-compat-report.md'
+ LOG_FILE: '/tmp/rustfs-s3-compat.log'
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+ exit 0
fi
+ TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+ EXISTING="$(gh issue list -R rustfs/backlog --state all \
+ --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+ --json number --jq '.[].number' || true)"
+ if [ -n "${EXISTING}" ]; then
+ echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+ exit 0
+ fi
+ redact() {
+ sed -E \
+ -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+ -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+ }
+ BODY_FILE="$(mktemp)"
+ {
+ echo "The **${SUITE_LABEL}** functional suite failed."
+ echo ""
+ echo "- Suite: \`${SUITE}\`"
+ echo "- Run: ${RUN_URL}"
+ echo "- Trigger: ${GITHUB_EVENT_NAME}"
+ echo "- Date: $(date -u +%Y-%m-%d)"
+ echo ""
+ echo "## Report (errors and symptoms)"
+ echo ""
+ if [ -s "${REPORT_FILE}" ]; then
+ redact < "${REPORT_FILE}"
+ elif [ -s "${LOG_FILE:-}" ]; then
+ echo "(report file missing; log tail below)"
+ echo ""
+ tail -n 200 "${LOG_FILE}" | redact
+ else
+ echo "(no report or log file was produced)"
+ fi
+ } | head -c 55000 > "${BODY_FILE}"
+ gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+ if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+ --body-file "${BODY_FILE}" --label functional-test; then
+ gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+ fi
+ echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
@@ -399,6 +318,24 @@ jobs:
'
done
+ - name: "Continue functional chain (next: KMS)"
+ # Only chain-triggered runs forward to the next suite; standalone
+ # workflow_dispatch runs stop after their own cleanup.
+ if: ${{ always() && github.event_name == 'repository_dispatch' }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+ exit 1
+ fi
+ echo "Dispatching next functional suite: KMS"
+ gh api --method POST repos/rustfs/rustfs/dispatches \
+ -f event_type='rustfs-chain-kms' \
+ -F 'client_payload[from_suite]=s3'
+
- name: Notify on failure
if: failure()
run: |
diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml
index 7b4d4595d..bb10f373a 100644
--- a/.github/workflows/rustfs-security-test.yml
+++ b/.github/workflows/rustfs-security-test.yml
@@ -46,10 +46,9 @@ on:
description: 'Reset the nodes after the test (DESTROYS test data/config)'
type: boolean
default: true
- workflow_run:
- # Runs last in the functional chain, after pool/heal, on the shared VMs.
- workflows: ["RustFS Pool Expansion / Heal Test"]
- types: [completed]
+ repository_dispatch:
+ # Chain handoff: dispatched when the pool expansion suite finishes (last link).
+ types: [rustfs-chain-security]
permissions:
contents: read
@@ -77,16 +76,27 @@ jobs:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360
- if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+ if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- - name: Checkout auto-testing scripts
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- with:
- repository: rustfs/auto-testing
- ref: main
- path: auto-testing
- persist-credentials: false
- token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ # auto-testing is private: clone it with the dedicated PF token (not
+ # GITHUB_TOKEN) and retry transient GitHub/network failures.
+ - name: Checkout auto-testing scripts (with retry)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ rm -rf auto-testing
+ for attempt in 1 2 3 4 5; do
+ if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+ echo "auto-testing cloned (attempt ${attempt})"
+ exit 0
+ fi
+ rm -rf auto-testing
+ echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+ sleep $((attempt * 15))
+ done
+ echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+ exit 1
- name: Checkout repository (for the OIDC live gate script)
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -193,155 +203,64 @@ jobs:
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
- cat > /tmp/rustfs-functional-index.html <<'EOF'
-
-
-
-
-
- RustFS Functional Test Reports
-
-
-
-
-
-
RustFS Functional Test Reports
-
Select a suite and date to view the build version used in that run.
-
-
-
-
-
-
-
-
- EOF
-
- INDEX_PATH="functional/index.html"
- INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
- INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
- if [ -n "${INDEX_SHA}" ]; then
- jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
- else
- jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
- '{message:$msg, content:$content}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+ - name: File failure issue in rustfs/backlog
+ if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ SUITE: 'security'
+ SUITE_LABEL: 'Security'
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ REPORT_FILE: '/tmp/rustfs-security-report.md'
+ LOG_FILE: ''
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+ exit 0
fi
+ TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+ EXISTING="$(gh issue list -R rustfs/backlog --state all \
+ --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+ --json number --jq '.[].number' || true)"
+ if [ -n "${EXISTING}" ]; then
+ echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+ exit 0
+ fi
+ redact() {
+ sed -E \
+ -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+ -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+ }
+ BODY_FILE="$(mktemp)"
+ {
+ echo "The **${SUITE_LABEL}** functional suite failed."
+ echo ""
+ echo "- Suite: \`${SUITE}\`"
+ echo "- Run: ${RUN_URL}"
+ echo "- Trigger: ${GITHUB_EVENT_NAME}"
+ echo "- Date: $(date -u +%Y-%m-%d)"
+ echo ""
+ echo "## Report (errors and symptoms)"
+ echo ""
+ if [ -s "${REPORT_FILE}" ]; then
+ redact < "${REPORT_FILE}"
+ elif [ -s "${LOG_FILE:-}" ]; then
+ echo "(report file missing; log tail below)"
+ echo ""
+ tail -n 200 "${LOG_FILE}" | redact
+ else
+ echo "(no report or log file was produced)"
+ fi
+ } | head -c 55000 > "${BODY_FILE}"
+ gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+ if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+ --body-file "${BODY_FILE}" --label functional-test; then
+ gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+ fi
+ echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml
index af6307848..242d6e831 100644
--- a/.github/workflows/rustfs-storage-test.yml
+++ b/.github/workflows/rustfs-storage-test.yml
@@ -20,10 +20,9 @@ on:
- single-multi
- multi-multi
default: all
- workflow_run:
- # Strict shared-environment order: run after tier test completes.
- workflows: ["RustFS Tier Test"]
- types: [completed]
+ repository_dispatch:
+ # Chain handoff: dispatched when the tier suite finishes.
+ types: [rustfs-chain-storage]
permissions:
contents: read
@@ -49,16 +48,27 @@ jobs:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 360
- if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+ if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- - name: Checkout auto-testing scripts
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- with:
- repository: rustfs/auto-testing
- ref: main
- path: auto-testing
- persist-credentials: false
- token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ # auto-testing is private: clone it with the dedicated PF token (not
+ # GITHUB_TOKEN) and retry transient GitHub/network failures.
+ - name: Checkout auto-testing scripts (with retry)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ rm -rf auto-testing
+ for attempt in 1 2 3 4 5; do
+ if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+ echo "auto-testing cloned (attempt ${attempt})"
+ exit 0
+ fi
+ rm -rf auto-testing
+ echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+ sleep $((attempt * 15))
+ done
+ echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+ exit 1
- name: Show environment
run: |
@@ -235,156 +245,64 @@ jobs:
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
- cat > /tmp/rustfs-functional-index.html <<'EOF'
-
-
-
-
-
- RustFS Functional Test Reports
-
-
-
-
-
-
RustFS Functional Test Reports
-
Select a suite and date to view the build version used in that run.
-
-
-
-
-
-
-
-
- EOF
-
- INDEX_PATH="functional/index.html"
- INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
- INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
- if [ -n "${INDEX_SHA}" ]; then
- jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
- else
- jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
- '{message:$msg, content:$content}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+ - name: File failure issue in rustfs/backlog
+ if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ SUITE: 'storage'
+ SUITE_LABEL: 'Storage engine'
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ REPORT_FILE: '/tmp/rustfs-storage-report.md'
+ LOG_FILE: '/tmp/rustfs-storage.log'
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+ exit 0
fi
+ TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+ EXISTING="$(gh issue list -R rustfs/backlog --state all \
+ --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+ --json number --jq '.[].number' || true)"
+ if [ -n "${EXISTING}" ]; then
+ echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+ exit 0
+ fi
+ redact() {
+ sed -E \
+ -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+ -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+ }
+ BODY_FILE="$(mktemp)"
+ {
+ echo "The **${SUITE_LABEL}** functional suite failed."
+ echo ""
+ echo "- Suite: \`${SUITE}\`"
+ echo "- Run: ${RUN_URL}"
+ echo "- Trigger: ${GITHUB_EVENT_NAME}"
+ echo "- Date: $(date -u +%Y-%m-%d)"
+ echo ""
+ echo "## Report (errors and symptoms)"
+ echo ""
+ if [ -s "${REPORT_FILE}" ]; then
+ redact < "${REPORT_FILE}"
+ elif [ -s "${LOG_FILE:-}" ]; then
+ echo "(report file missing; log tail below)"
+ echo ""
+ tail -n 200 "${LOG_FILE}" | redact
+ else
+ echo "(no report or log file was produced)"
+ fi
+ } | head -c 55000 > "${BODY_FILE}"
+ gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+ if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+ --body-file "${BODY_FILE}" --label functional-test; then
+ gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+ fi
+ echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
@@ -415,6 +333,24 @@ jobs:
'
done
+ - name: "Continue functional chain (next: Heal)"
+ # Only chain-triggered runs forward to the next suite; standalone
+ # workflow_dispatch runs stop after their own cleanup.
+ if: ${{ always() && github.event_name == 'repository_dispatch' }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+ exit 1
+ fi
+ echo "Dispatching next functional suite: Heal"
+ gh api --method POST repos/rustfs/rustfs/dispatches \
+ -f event_type='rustfs-chain-heal' \
+ -F 'client_payload[from_suite]=storage'
+
- name: Notify on failure
if: failure()
run: |
diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml
index 059995c45..07cf060ad 100644
--- a/.github/workflows/rustfs-tier-test.yml
+++ b/.github/workflows/rustfs-tier-test.yml
@@ -20,10 +20,9 @@ on:
required: false
default: false
type: boolean
- workflow_run:
- # Strict shared-environment order: run after KMS test completes.
- workflows: ["RustFS KMS Test"]
- types: [completed]
+ repository_dispatch:
+ # Chain handoff: dispatched when the KMS suite finishes.
+ types: [rustfs-chain-tier]
permissions:
contents: read
@@ -49,17 +48,31 @@ env:
jobs:
tier-test:
runs-on: smoke-testing
+ # Requirement: a failing suite must not fail the workflow; failures
+ # are filed to rustfs/backlog and the chain continues.
+ continue-on-error: true
timeout-minutes: 420
- if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
+ if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- - name: Checkout auto-testing scripts
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- with:
- repository: rustfs/auto-testing
- ref: main
- path: auto-testing
- persist-credentials: false
- token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ # auto-testing is private: clone it with the dedicated PF token (not
+ # GITHUB_TOKEN) and retry transient GitHub/network failures.
+ - name: Checkout auto-testing scripts (with retry)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ rm -rf auto-testing
+ for attempt in 1 2 3 4 5; do
+ if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+ echo "auto-testing cloned (attempt ${attempt})"
+ exit 0
+ fi
+ rm -rf auto-testing
+ echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+ sleep $((attempt * 15))
+ done
+ echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+ exit 1
- name: Show environment
run: |
@@ -241,105 +254,64 @@ jobs:
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
- cat > /tmp/rustfs-functional-index.html <<'EOF'
-
-
-
-
-
- RustFS Functional Test Reports
-
-
-
-
-
-
RustFS Functional Test Reports
-
S3, KMS, Tier report tabs. Each tab lists reports by date.
-
-
-
-
-
-
-
- EOF
-
- INDEX_PATH="functional/index.html"
- INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
- INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
- if [ -n "${INDEX_SHA}" ]; then
- jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
- else
- jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
- '{message:$msg, content:$content}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+ - name: File failure issue in rustfs/backlog
+ if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ SUITE: 'tier'
+ SUITE_LABEL: 'Tier'
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ REPORT_FILE: '/tmp/rustfs-tier-report.md'
+ LOG_FILE: '/tmp/rustfs-tier.log'
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+ exit 0
fi
+ TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+ EXISTING="$(gh issue list -R rustfs/backlog --state all \
+ --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+ --json number --jq '.[].number' || true)"
+ if [ -n "${EXISTING}" ]; then
+ echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+ exit 0
+ fi
+ redact() {
+ sed -E \
+ -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+ -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+ }
+ BODY_FILE="$(mktemp)"
+ {
+ echo "The **${SUITE_LABEL}** functional suite failed."
+ echo ""
+ echo "- Suite: \`${SUITE}\`"
+ echo "- Run: ${RUN_URL}"
+ echo "- Trigger: ${GITHUB_EVENT_NAME}"
+ echo "- Date: $(date -u +%Y-%m-%d)"
+ echo ""
+ echo "## Report (errors and symptoms)"
+ echo ""
+ if [ -s "${REPORT_FILE}" ]; then
+ redact < "${REPORT_FILE}"
+ elif [ -s "${LOG_FILE:-}" ]; then
+ echo "(report file missing; log tail below)"
+ echo ""
+ tail -n 200 "${LOG_FILE}" | redact
+ else
+ echo "(no report or log file was produced)"
+ fi
+ } | head -c 55000 > "${BODY_FILE}"
+ gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+ if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+ --body-file "${BODY_FILE}" --label functional-test; then
+ gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+ fi
+ echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
@@ -399,6 +371,24 @@ jobs:
fi
[ "${failed}" -eq 0 ]
+ - name: "Continue functional chain (next: Storage engine)"
+ # Only chain-triggered runs forward to the next suite; standalone
+ # workflow_dispatch runs stop after their own cleanup.
+ if: ${{ always() && github.event_name == 'repository_dispatch' }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+ exit 1
+ fi
+ echo "Dispatching next functional suite: Storage engine"
+ gh api --method POST repos/rustfs/rustfs/dispatches \
+ -f event_type='rustfs-chain-storage' \
+ -F 'client_payload[from_suite]=tier'
+
- name: Notify on failure
if: failure()
run: |
diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml
index 769490b40..60a936c88 100644
--- a/.github/workflows/rustfs-upgrade-test.yml
+++ b/.github/workflows/rustfs-upgrade-test.yml
@@ -53,11 +53,9 @@ on:
description: 'Reset the nodes after the test (DESTROYS test data/config)'
type: boolean
default: true
- workflow_run:
- # Runs first in the functional chain: upgrade compatibility gates the
- # nightly suites that follow (S3 -> KMS -> Tier -> Pool/Heal -> Security).
- workflows: ["Nightly GNU Build"]
- types: [completed]
+ repository_dispatch:
+ # Functional-chain entry: dispatched by rustfs-functional-chain.yml.
+ types: [rustfs-chain-upgrade]
permissions:
contents: read
@@ -83,16 +81,27 @@ jobs:
runs-on: smoke-testing
continue-on-error: true
timeout-minutes: 420
- if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
+ if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
steps:
- - name: Checkout auto-testing scripts
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- with:
- repository: rustfs/auto-testing
- ref: main
- path: auto-testing
- persist-credentials: false
- token: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ # auto-testing is private: clone it with the dedicated PF token (not
+ # GITHUB_TOKEN) and retry transient GitHub/network failures.
+ - name: Checkout auto-testing scripts (with retry)
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ rm -rf auto-testing
+ for attempt in 1 2 3 4 5; do
+ if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
+ echo "auto-testing cloned (attempt ${attempt})"
+ exit 0
+ fi
+ rm -rf auto-testing
+ echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
+ sleep $((attempt * 15))
+ done
+ echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
+ exit 1
- name: Show environment
run: |
@@ -288,155 +297,64 @@ jobs:
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
fi
- cat > /tmp/rustfs-functional-index.html <<'EOF'
-
-
-
-
-
- RustFS Functional Test Reports
-
-
-
-
-
-
RustFS Functional Test Reports
-
Select a suite and date to view the build version used in that run.
-
-
-
-
-
-
-
-
- EOF
-
- INDEX_PATH="functional/index.html"
- INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
- INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
- if [ -n "${INDEX_SHA}" ]; then
- jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
- '{message:$msg, content:$content, sha:$sha}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
- else
- jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
- '{message:$msg, content:$content}' \
- | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
+ - name: File failure issue in rustfs/backlog
+ if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ SUITE: 'upgrade'
+ SUITE_LABEL: 'Upgrade compatibility'
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ REPORT_FILE: '/tmp/rustfs-upgrade-report.md'
+ LOG_FILE: '/tmp/rustfs-upgrade.log'
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
+ exit 0
fi
+ TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
+ EXISTING="$(gh issue list -R rustfs/backlog --state all \
+ --search "in:title \"run ${GITHUB_RUN_ID}\"" \
+ --json number --jq '.[].number' || true)"
+ if [ -n "${EXISTING}" ]; then
+ echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
+ exit 0
+ fi
+ redact() {
+ sed -E \
+ -e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
+ -e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
+ -e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
+ }
+ BODY_FILE="$(mktemp)"
+ {
+ echo "The **${SUITE_LABEL}** functional suite failed."
+ echo ""
+ echo "- Suite: \`${SUITE}\`"
+ echo "- Run: ${RUN_URL}"
+ echo "- Trigger: ${GITHUB_EVENT_NAME}"
+ echo "- Date: $(date -u +%Y-%m-%d)"
+ echo ""
+ echo "## Report (errors and symptoms)"
+ echo ""
+ if [ -s "${REPORT_FILE}" ]; then
+ redact < "${REPORT_FILE}"
+ elif [ -s "${LOG_FILE:-}" ]; then
+ echo "(report file missing; log tail below)"
+ echo ""
+ tail -n 200 "${LOG_FILE}" | redact
+ else
+ echo "(no report or log file was produced)"
+ fi
+ } | head -c 55000 > "${BODY_FILE}"
+ gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
+ if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
+ --body-file "${BODY_FILE}" --label functional-test; then
+ gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
+ fi
+ echo "filed backlog issue for suite ${SUITE}"
- name: Upload report and logs
if: always()
@@ -468,6 +386,24 @@ jobs:
'
done
+ - name: "Continue functional chain (next: S3 compatibility)"
+ # Only chain-triggered runs forward to the next suite; standalone
+ # workflow_dispatch runs stop after their own cleanup.
+ if: ${{ always() && github.event_name == 'repository_dispatch' }}
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+ run: |
+ set -euo pipefail
+ if [ -z "${GH_TOKEN:-}" ]; then
+ echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
+ exit 1
+ fi
+ echo "Dispatching next functional suite: S3 compatibility"
+ gh api --method POST repos/rustfs/rustfs/dispatches \
+ -f event_type='rustfs-chain-s3' \
+ -F 'client_payload[from_suite]=upgrade'
+
- name: Notify on failure
if: failure()
run: |