rustfs/scripts/test_scanner_validation_harness.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
SCRIPT="$ROOT_DIR/scripts/run_scanner_validation_harness.sh"
TMP_DIR=$(mktemp -d)
trap 'rm -rf "$TMP_DIR"' EXIT

BIN_DIR="$TMP_DIR/bin"
OUT_DIR="$TMP_DIR/out"
mkdir -p "$BIN_DIR"

cat >"$BIN_DIR/mc" <<'STUB'
#!/usr/bin/env bash
set -euo pipefail

printf '%s\n' "$*" >>"${MC_LOG:?}"
if [[ "$*" == "admin config get rustfs-local scanner" ]]; then
  printf 'scanner delay="30" max_wait="15"\n'
elif [[ "$*" == "admin config get rustfs-local heal" ]]; then
  printf 'heal bitrot_cycle="2592000"\n'
else
  printf 'unexpected mc args: %s\n' "$*" >&2
  exit 1
fi
STUB

cat >"$BIN_DIR/awscurl" <<'STUB'
#!/usr/bin/env bash
set -euo pipefail

printf '%s\n' "$*" >>"${AWSCURL_LOG:?}"
printf 'access-env-present=%s\n' "${AWS_ACCESS_KEY_ID:+yes}" >>"${AWSCURL_LOG:?}"
printf 'secret-env-present=%s\n' "${AWS_SECRET_ACCESS_KEY:+yes}" >>"${AWSCURL_LOG:?}"

url="${*: -1}"
if [[ "$url" == *"/rustfs/admin/v3/background-heal/status" ]]; then
  cat <<'JSON'
{
  "healQueueLength": 4,
  "healActiveTasks": 1,
  "healOperations": {
    "queueLength": 4,
    "activeTasks": 1,
    "queuedBySource": {
      "scanner": 2,
      "admin": 1,
      "autoHeal": 1,
      "internal": 0
    },
    "activeBySource": {
      "scanner": 1,
      "admin": 0,
      "autoHeal": 0,
      "internal": 0
    },
    "queuedByPriority": {
      "low": 2,
      "normal": 1,
      "high": 1,
      "urgent": 0
    },
    "activeByPriority": {
      "low": 1,
      "normal": 0,
      "high": 0,
      "urgent": 0
    }
  }
}
JSON
  exit 0
fi

if [[ "$url" == *"/rustfs/admin/v3/metrics?types=1&by-host=true&n=1" ]]; then
  cat <<'JSON'
{"node":"node-a","metrics":{"scanner":{"maintenanceControl":{"primaryControl":"active"}}}}
JSON
  exit 0
fi

cat <<'JSON'
{
  "runtime_config": {
    "delay": { "value": 30, "source": "config" }
  },
  "metrics": {
    "life_time_ops": {
      "scan_cycle": 9,
      "scan_bucket_drive": 11,
      "scan_object": 13,
      "save_usage": 5
    },
    "pacing_pressure": {
      "primary_pressure": "active_scans"
    },
    "current_cycle_objects_scanned": 5,
    "current_cycle_directories_scanned": 2,
    "last_cycle_result": "success",
    "last_cycle_partial_reason": "",
    "last_cycle_partial_source": "",
    "lifecycle_transition": {
      "scanner_missed": 0
    },
    "current_cycle_usage_saves": 2,
    "last_cycle_usage_saves": 3,
    "usage_freshness": {
      "dirty_pending_buckets": 1,
      "last_dirty_mark_unix_secs": 1800000001,
      "last_dirty_clear_unix_secs": 1800000002,
      "last_cycle_dirty_buckets": 1,
      "last_cycle_cleared_dirty_buckets": 1,
      "last_usage_save_unix_secs": 1800000003,
      "last_usage_save_result": "success",
      "last_usage_save_result_code": 1
    },
    "source_work": [
      { "source": "usage", "missed": 0 },
      { "source": "lifecycle", "missed": 0 }
    ]
  }
}
JSON
STUB

cat >"$BIN_DIR/pidof" <<'STUB'
#!/usr/bin/env bash
exit 1
STUB

cat >"$BIN_DIR/iostat" <<'STUB'
#!/usr/bin/env bash
printf 'iostat sample\n'
STUB

cat >"$BIN_DIR/mpstat" <<'STUB'
#!/usr/bin/env bash
printf 'mpstat sample\n'
STUB

cat >"$BIN_DIR/jq" <<'STUB'
#!/usr/bin/env bash
set -euo pipefail

if [[ "$1" == "." ]]; then
  cat
  exit 0
fi

if [[ "$1" == "-s" ]]; then
  cat <<'JSON'
{
  "healOperations": {
    "queueLength": 8,
    "activeTasks": 2,
    "queuedBySource": {
      "scanner": 4,
      "admin": 2,
      "autoHeal": 2,
      "internal": 0
    }
  }
}
JSON
  exit 0
fi

if [[ "$1" == "-r" ]]; then
  while [[ $# -gt 0 ]]; do
    if [[ "$1" == "--arg" && "${2:-}" == "ts" ]]; then
      printf '"%s","active_scans",5,2,"success","","",0,0,2,3,1,1,1,"success",1800000003,9,11,13,5,8,2,4,2,2\n' "$3"
      exit 0
    fi
    shift
  done
fi

printf 'unexpected jq args: %s\n' "$*" >&2
exit 1
STUB

chmod +x "$BIN_DIR/mc" "$BIN_DIR/awscurl" "$BIN_DIR/pidof" "$BIN_DIR/iostat" "$BIN_DIR/mpstat" "$BIN_DIR/jq"

mc_log="$TMP_DIR/mc.log"
awscurl_log="$TMP_DIR/awscurl.log"
RUSTFS_SECRET_KEY=rustfsadmin MC_LOG="$mc_log" AWSCURL_LOG="$awscurl_log" PATH="$BIN_DIR:$PATH" "$SCRIPT" \
  --alias rustfs-local \
  --endpoint http://127.0.0.1:9000 \
  --access-key rustfsadmin \
  --deployment single-disk \
  --workload-label small-object-idle \
  --metrics-endpoints http://node-a:9000,http://node-b:9000 \
  --samples 2 \
  --interval-secs 0 \
  --out-dir "$OUT_DIR" \
  --skip-host-telemetry

test -s "$OUT_DIR/scanner-config.before.txt"
test -s "$OUT_DIR/heal-config.before.txt"
grep -q 'scanner delay="30" max_wait="15"' "$OUT_DIR/scanner-config.before.txt"
grep -q 'heal bitrot_cycle="2592000"' "$OUT_DIR/heal-config.before.txt"

status_count=$(find "$OUT_DIR/status" -type f -name 'scanner-status.*.json' | wc -l | tr -d ' ')
if [[ "$status_count" != "2" ]]; then
  echo "Expected 2 scanner status snapshots, got $status_count" >&2
  exit 1
fi

heal_status_count=$(find "$OUT_DIR/heal" -type f -name 'background-heal-status.*.json' | wc -l | tr -d ' ')
if [[ "$heal_status_count" != "4" ]]; then
  echo "Expected 4 background heal status snapshots, got $heal_status_count" >&2
  exit 1
fi

metrics_count=$(find "$OUT_DIR/metrics" -type f -name 'admin-metrics.*.ndjson' | wc -l | tr -d ' ')
if [[ "$metrics_count" != "4" ]]; then
  echo "Expected 4 distributed admin metrics snapshots, got $metrics_count" >&2
  exit 1
fi

test -s "$OUT_DIR/scanner-summary.csv"
if [[ "$(wc -l <"$OUT_DIR/scanner-summary.csv" | tr -d ' ')" != "3" ]]; then
  echo "Expected scanner summary header plus 2 rows" >&2
  exit 1
fi
grep -q 'current_cycle_usage_saves,last_cycle_usage_saves,usage_dirty_pending_buckets' "$OUT_DIR/scanner-summary.csv"
grep -q 'life_time_scan_cycle,life_time_scan_bucket_drive,life_time_scan_object,life_time_save_usage' "$OUT_DIR/scanner-summary.csv"
grep -q 'heal_queue_length,heal_active_tasks,heal_scanner_queued,heal_admin_queued,heal_auto_heal_queued' "$OUT_DIR/scanner-summary.csv"
grep -q '"active_scans",5,2,"success","","",0,0,2,3,1,1,1,"success",1800000003,9,11,13,5,8,2,4,2,2' "$OUT_DIR/scanner-summary.csv"

grep -q '^deployment=single-disk$' "$OUT_DIR/run-metadata.env"
grep -q '^workload_label=small-object-idle$' "$OUT_DIR/run-metadata.env"
grep -q '^metrics_endpoints=http://node-a:9000,http://node-b:9000$' "$OUT_DIR/run-metadata.env"
grep -q '## Scanner Validation Report' "$OUT_DIR/scanner-validation-report.md"
grep -q 'Distributed admin metrics snapshots: 4' "$OUT_DIR/scanner-validation-report.md"
grep -q 'Background heal status snapshots: 4' "$OUT_DIR/scanner-validation-report.md"
grep -q 'Usage freshness summary:' "$OUT_DIR/scanner-validation-report.md"
grep -q 'admin config get rustfs-local scanner' "$mc_log"
grep -q 'admin config get rustfs-local heal' "$mc_log"
grep -q -- '--request GET' "$awscurl_log"
grep -q -- '--request POST' "$awscurl_log"
grep -q -- 'access-env-present=yes' "$awscurl_log"
grep -q -- 'secret-env-present=yes' "$awscurl_log"
if grep -q -- '--secret_key' "$awscurl_log"; then
  echo "Expected awscurl to receive the secret through the environment, not argv" >&2
  exit 1
fi
grep -q -- 'http://127.0.0.1:9000/rustfs/admin/v3/scanner/status' "$awscurl_log"
grep -q -- 'http://node-a:9000/rustfs/admin/v3/background-heal/status' "$awscurl_log"
grep -q -- 'http://node-b:9000/rustfs/admin/v3/background-heal/status' "$awscurl_log"
grep -q -- 'http://node-a:9000/rustfs/admin/v3/metrics?types=1&by-host=true&n=1' "$awscurl_log"
grep -q -- 'http://node-b:9000/rustfs/admin/v3/metrics?types=1&by-host=true&n=1' "$awscurl_log"

missing_pid_out="$TMP_DIR/out-missing-pid"
RUSTFS_SECRET_KEY=rustfsadmin MC_LOG="$mc_log" AWSCURL_LOG="$awscurl_log" PATH="$BIN_DIR:$PATH" "$SCRIPT" \
  --alias rustfs-local \
  --endpoint http://127.0.0.1:9000 \
  --access-key rustfsadmin \
  --deployment single-disk \
  --workload-label missing-pid \
  --samples 1 \
  --interval-secs 1 \
  --out-dir "$missing_pid_out"

test -s "$missing_pid_out/status/scanner-status.1."*.json
test -s "$missing_pid_out/iostat.txt"
test -s "$missing_pid_out/mpstat.txt"

missing_args_log="$TMP_DIR/missing-args.log"
if PATH="$BIN_DIR:$PATH" "$SCRIPT" --alias rustfs-local >"$missing_args_log" 2>&1; then
  echo "Expected missing required arguments to fail" >&2
  exit 1
fi

grep -q -- '--alias, --endpoint, and RUSTFS_ACCESS_KEY (or --access-key) are required' "$missing_args_log"

missing_secret_log="$TMP_DIR/missing-secret.log"
if PATH="$BIN_DIR:$PATH" "$SCRIPT" \
  --alias rustfs-local \
  --endpoint http://127.0.0.1:9000 \
  --access-key rustfsadmin >"$missing_secret_log" 2>&1; then
  echo "Expected missing RUSTFS_SECRET_KEY to fail" >&2
  exit 1
fi

grep -q -- 'RUSTFS_SECRET_KEY is required for scanner status requests' "$missing_secret_log"

invalid_secret_env_log="$TMP_DIR/invalid-secret-env.log"
if PATH="$BIN_DIR:$PATH" "$SCRIPT" \
  --alias rustfs-local \
  --endpoint http://127.0.0.1:9000 \
  --access-key rustfsadmin \
  --secret-key-env 'not-valid!' >"$invalid_secret_env_log" 2>&1; then
  echo "Expected invalid --secret-key-env to fail" >&2
  exit 1
fi

grep -q -- '--secret-key-env must be a valid environment variable name' "$invalid_secret_env_log"

missing_value_log="$TMP_DIR/missing-value.log"
if PATH="$BIN_DIR:$PATH" "$SCRIPT" --alias >"$missing_value_log" 2>&1; then
  echo "Expected --alias without a value to fail" >&2
  exit 1
fi

grep -q -- 'missing value for --alias' "$missing_value_log"

secret_arg_log="$TMP_DIR/secret-arg.log"
if PATH="$BIN_DIR:$PATH" "$SCRIPT" --secret-key rustfsadmin >"$secret_arg_log" 2>&1; then
  echo "Expected --secret-key argv to be rejected" >&2
  exit 1
fi

grep -q -- 'unknown arg: --secret-key' "$secret_arg_log"