mirror of
https://github.com/nearai/ironclaw.git
synced 2026-09-03 08:06:01 +08:00
test(live): fix tool-name assertions + re-record zizmor traces
The two `zizmor_scan*` live tests had four broken tool-name assertions
that silently failed to match: `tools.iter().any(|t| t == "shell")`
against a tool list that now contains `"shell(cmd)"` strings (tool
events carry args via `format_action_display_name` in
`src/bridge/router.rs`). Two of the four were negative assertions
checking for the absence of `tool_install` recovery loops — those
silently passed even when a recovery loop actually ran. `sandbox_live_e2e.rs:203`
already used the correct `t == "shell" || t.starts_with("shell(")`
pattern; applied it consistently to all four sites.
Verified live:
- `IRONCLAW_LIVE_TEST=1 cargo test --test e2e_live -- zizmor_scan --ignored --test-threads=1`
→ 2 passed, 0 failed, 51.78s. Agent installs and runs zizmor
end-to-end, producing real findings (exit code 14, dangerous
triggers, excessive permissions, etc.).
Traces re-recorded with the tuned postamble (commit 50d85175) and
scrubbed: replaced `/home/illia/.cargo/bin/zizmor` with
`/home/user/.cargo/bin/zizmor` per the developer-local-path ban in
`.claude/rules/review-discipline.md`. No credentials, PII, or
high-entropy secrets in either trace (only git SHAs from zizmor's
workflow analysis output).
Replay still passes: `cargo test --test e2e_live -- zizmor_scan --ignored`
→ 2/2 ok.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -53,8 +53,13 @@ mod live_tests {
|
||||
);
|
||||
|
||||
// The agent should have used the shell tool to install/run zizmor.
|
||||
// Tool events now carry args (e.g. `"shell(cmd)"`) via
|
||||
// `format_action_display_name` in `src/bridge/router.rs`, so match both
|
||||
// the bare name and the argument-prefixed form.
|
||||
assert!(
|
||||
tools.iter().any(|t| t == "shell"),
|
||||
tools
|
||||
.iter()
|
||||
.any(|t| t == "shell" || t.starts_with("shell(")),
|
||||
"Expected shell tool to be used for running zizmor, got: {tools:?}"
|
||||
);
|
||||
|
||||
@@ -131,10 +136,14 @@ mod live_tests {
|
||||
|
||||
// V2 without auto-approve hits an approval gate for shell/tool_install.
|
||||
// The response may be the approval prompt itself rather than agent output.
|
||||
// Verify the agent at least attempted a relevant action.
|
||||
// Verify the agent at least attempted a relevant action. Tool events
|
||||
// carry args (e.g. `"shell(cmd)"`) via `format_action_display_name`, so
|
||||
// accept either the bare name or the argument-prefixed form.
|
||||
let attempted_relevant_tool = tools.iter().any(|t| {
|
||||
t == "shell"
|
||||
|| t.starts_with("shell(")
|
||||
|| t == "tool_install"
|
||||
|| t.starts_with("tool_install(")
|
||||
|| t.starts_with("tool_search")
|
||||
|| t.starts_with("skill_search")
|
||||
});
|
||||
@@ -399,10 +408,18 @@ mod live_tests {
|
||||
|
||||
// The agent must NOT have run a tool_install / tool_activate
|
||||
// recovery loop — that's the bad behaviour the post-flight
|
||||
// detector eliminates.
|
||||
let bad_recovery = phase_a_tools
|
||||
.iter()
|
||||
.any(|t| t == "tool_install" || t == "tool_activate" || t == "tool-install");
|
||||
// detector eliminates. Match both bare names and the
|
||||
// argument-prefixed `"<name>(args)"` form emitted by
|
||||
// `format_action_display_name`; an exact-match check would silently
|
||||
// miss `"tool_install(foo)"` and turn this into a false negative.
|
||||
let bad_recovery = phase_a_tools.iter().any(|t| {
|
||||
t == "tool_install"
|
||||
|| t.starts_with("tool_install(")
|
||||
|| t == "tool_activate"
|
||||
|| t.starts_with("tool_activate(")
|
||||
|| t == "tool-install"
|
||||
|| t.starts_with("tool-install(")
|
||||
});
|
||||
assert!(
|
||||
!bad_recovery,
|
||||
"Phase A: agent ran a tool_install/tool_activate recovery loop instead \
|
||||
@@ -507,11 +524,16 @@ mod live_tests {
|
||||
"[DriveAuthGate][Phase B] Tools attempted ({}): {phase_b_tools:?}",
|
||||
phase_b_tools.len()
|
||||
);
|
||||
// Match bare and argument-prefixed names; see the Phase A comment.
|
||||
let phase_b_recovery = phase_b_tools.iter().any(|t| {
|
||||
t == "tool_install"
|
||||
|| t.starts_with("tool_install(")
|
||||
|| t == "tool-install"
|
||||
|| t.starts_with("tool-install(")
|
||||
|| t == "tool_activate"
|
||||
|| t.starts_with("tool_activate(")
|
||||
|| t == "secret_list"
|
||||
|| t.starts_with("secret_list(")
|
||||
|| t.starts_with("tool_search")
|
||||
});
|
||||
assert!(
|
||||
|
||||
172
tests/fixtures/llm_traces/live/zizmor_scan.json
vendored
172
tests/fixtures/llm_traces/live/zizmor_scan.json
vendored
File diff suppressed because one or more lines are too long
189
tests/fixtures/llm_traces/live/zizmor_scan.log
vendored
189
tests/fixtures/llm_traces/live/zizmor_scan.log
vendored
@@ -1,135 +1,84 @@
|
||||
# Live Test Session: zizmor_scan
|
||||
# Mode: Replay
|
||||
# LLM calls: 8, Input tokens: 143884, Output tokens: 1410
|
||||
# Wall time: 0.9s, Cost: $0.0000
|
||||
# Mode: Live
|
||||
# LLM calls: 9, Input tokens: 223223, Output tokens: 1425
|
||||
# Wall time: 32.3s, Cost: $0.0000
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
› can we run https://github.com/zizmorcore/zizmor
|
||||
○ Processing...
|
||||
○ Thinking (step 1)...
|
||||
○ Running tool_search...
|
||||
● tool_search
|
||||
✓ tool_search
|
||||
tool_search → {
|
||||
"count": 0,
|
||||
"results": [],
|
||||
"searched_online": false
|
||||
}
|
||||
○ Thinking (step 2)...
|
||||
○ Running command...
|
||||
● shell
|
||||
✓ shell
|
||||
shell → {
|
||||
"exit_code": 0,
|
||||
"output": "/Users/coder/.cargo/bin/zizmor\n",
|
||||
"sandboxed": false,
|
||||
"success": true
|
||||
}
|
||||
○ Thinking (step 3)...
|
||||
○ Running list_dir...
|
||||
● list_dir
|
||||
✓ list_dir
|
||||
list_dir → {
|
||||
"count": 70,
|
||||
"entries": [
|
||||
".claude/",
|
||||
".git/",
|
||||
".githooks/",
|
||||
".github/",
|
||||
"channels-src/",
|
||||
"crates/",
|
||||
"deploy/",
|
||||
"docker/",
|
||||
"docs/",
|
||||
"fuzz/",
|
||||
"migratio…
|
||||
○ Thinking (step 4)...
|
||||
○ Running list_dir...
|
||||
● list_dir
|
||||
✓ list_dir
|
||||
list_dir → {
|
||||
"count": 4,
|
||||
"entries": [
|
||||
"scripts/",
|
||||
"workflows/",
|
||||
"labeler.yml (3.4KB)",
|
||||
"pull_request_template.md (1.7KB)"
|
||||
],
|
||||
"path": "/Users/coder/ironclaw8/.github",
|
||||
"truncated": false…
|
||||
○ Thinking (step 5)...
|
||||
○ Running list_dir...
|
||||
● list_dir
|
||||
✓ list_dir
|
||||
list_dir → {
|
||||
"count": 14,
|
||||
"entries": [
|
||||
"claude-review.yml (5.3KB)",
|
||||
"code_style.yml (3.3KB)",
|
||||
"coverage.yml (6.9KB)",
|
||||
"docker.yml (4.3KB)",
|
||||
"e2e.yml (4.0KB)",
|
||||
"pr-label-classify.yml (…
|
||||
○ Thinking (step 6)...
|
||||
○ Running command...
|
||||
● shell
|
||||
✓ shell
|
||||
shell → {
|
||||
"exit_code": 2,
|
||||
"output": "error: invalid value 'pretty' for '--format <FORMAT>'\n [possible values: plain, json, json-v1, sarif, github]\n\nFor more information, try '--help'.\n",
|
||||
"sandboxed…
|
||||
○ Thinking (step 7)...
|
||||
○ Running command...
|
||||
● shell
|
||||
✓ shell
|
||||
shell → {
|
||||
"exit_code": 14,
|
||||
"output": "help[artipacked]: credential persistence through GitHub Actions artifacts\n --> .github/workflows/claude-review.yml:23:9\n |\n23 | - uses: actions/checkout…
|
||||
○ Thinking (step 8)...
|
||||
… Done
|
||||
## Turn 1
|
||||
[USER] › can we run https://github.com/zizmorcore/zizmor
|
||||
────────────────────────────────────────────────────
|
||||
Done! I ran zizmor on your GitHub Actions workflows and found **security issues across all 14 workflow files**. Here's the summary:
|
||||
|
||||
## 🔴 Critical Issues Found
|
||||
|
||||
### 1. **Unpinned Action References** (Most Common)
|
||||
Every workflow uses actions like `actions/checkout@v6` instead of pinned SHA hashes. This is a supply chain risk - the action could change without notice.
|
||||
Yes, zizmor can be run! It's already installed (v1.24.1) and I just ran it on the repository's GitHub workflows. Here's what it found:
|
||||
|
||||
**Example fixes needed:**
|
||||
```yaml
|
||||
# Before
|
||||
uses: actions/checkout@v6
|
||||
## Summary
|
||||
|
||||
# After
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v6
|
||||
```
|
||||
**Exit code 14** indicates security issues were detected.
|
||||
|
||||
### 2. **Dangerous Triggers**
|
||||
- `pr-label-classify.yml`, `pr-label-scope.yml`, `release-plz-batch-summary.yml` use `pull_request_target` which is fundamentally insecure when combined with PR code execution.
|
||||
### Critical Findings
|
||||
|
||||
### 3. **Template Injection Risks**
|
||||
Multiple workflows have code injection vulnerabilities where user-controlled input can be expanded into shell commands:
|
||||
- `docker.yml` - workflow inputs directly in run blocks
|
||||
- `release.yml` - matrix args and tags in commands
|
||||
- `regression-test-check.yml` - PR base ref in git commands
|
||||
#### 1. **Dangerous Triggers** (High Severity, Medium Confidence)
|
||||
- **`.github/workflows/pr-label-classify.yml`** - Uses `pull_request_target` trigger
|
||||
- **`.github/workflows/pr-label-scope.yml`** - Uses `pull_request_target` trigger
|
||||
- **`.github/workflows/release-plz-batch-summary.yml`** - Uses `pull_request_target` trigger
|
||||
|
||||
### 4. **Excessive Permissions**
|
||||
- `coverage.yml` has `id-token: write` at workflow level (too broad)
|
||||
- `release.yml` has `contents: write` at workflow level
|
||||
- Most workflows use default permissions instead of explicit least-privilege
|
||||
`pull_request_target` is fundamentally insecure because it runs in the context of the base branch with write permissions, making it vulnerable to code injection from PRs.
|
||||
|
||||
### 5. **Secrets Outside Environments**
|
||||
`release-plz.yml` and `claude-review.yml` access secrets (API keys, private keys) without using dedicated GitHub Environments for protection.
|
||||
#### 2. **Template Injection** (High Severity, High Confidence)
|
||||
Multiple workflows have unsafe template expansions that could allow code injection:
|
||||
|
||||
### 6. **Artipacked (Credential Persistence)**
|
||||
Nearly all `actions/checkout` steps don't set `persist-credentials: false`, leaving Git credentials in the runner artifact.
|
||||
- **`.github/workflows/code_style.yml`** - Lines 83 (`github.base_ref`), 271 (`needs.changes.outputs.has_code`)
|
||||
- **`.github/workflows/rebuild-release-image.yml`** - Lines 114-118 (`inputs.source_ref`, `inputs.tag`, `steps.*.outputs.*`)
|
||||
|
||||
## 📊 Severity Breakdown
|
||||
- **Errors**: 70+ (unpinned uses, dangerous triggers, template injection)
|
||||
- **Warnings**: 20+ (excessive permissions, secrets-outside-env)
|
||||
- **Help/Info**: 30+ (artipacked, superfluous-actions, template-injection)
|
||||
These inject user-controllable data directly into shell commands via `${{ }}` expressions.
|
||||
|
||||
Would you like me to:
|
||||
1. Generate a SARIF report for GitHub Security tab integration?
|
||||
2. Create specific PRs to fix the highest-priority issues?
|
||||
3. Focus on one workflow file at a time?
|
||||
#### 3. **Template Injection** (Informational Severity, Low Confidence)
|
||||
Several `GITHUB_STEP_SUMMARY` injections in:
|
||||
- `.github/workflows/docker.yml` - Metadata output steps
|
||||
- `.github/workflows/rebuild-release-image.yml` - Summary generation
|
||||
|
||||
### Recommendation
|
||||
|
||||
1. **High Priority**: Replace `pull_request_target` with safer alternatives like `pull_request` + approved checks
|
||||
2. **Medium Priority**: Sanitize template injections by using environment variables or avoiding direct expansion of user input
|
||||
3. **Consider**: Running zizmor as a CI check on all PRs to catch new issues
|
||||
|
||||
Would you like me to analyze specific workflow files in more detail or suggest fixes for these issues?
|
||||
|
||||
## Activity
|
||||
○ Processing...
|
||||
○ Calling LLM...
|
||||
○ Step complete — 19610 in / 104 out tokens
|
||||
● shell(which zizmor || cargo install --version 1.11.1 zizmor 2>&1 |...)
|
||||
✓ shell(which zizmor || cargo install --version 1.11.1 zizmor 2>&1 |...)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 19717 in / 127 out tokens
|
||||
● shell(cd /tmp && git clone --depth 1 https://github.com/zizmorcore...)
|
||||
✓ shell(cd /tmp && git clone --depth 1 https://github.com/zizmorcore...)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 19827 in / 91 out tokens
|
||||
● shell(zizmor --version)
|
||||
✓ shell(zizmor --version)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 19905 in / 152 out tokens
|
||||
● shell(ls -la /tmp/zizmor)
|
||||
✓ shell(ls -la /tmp/zizmor)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 20012 in / 125 out tokens
|
||||
● shell(/tmp/zizmor --help)
|
||||
✓ shell(/tmp/zizmor --help)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 21614 in / 103 out tokens
|
||||
● glob(.)
|
||||
✓ glob(.)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 21776 in / 92 out tokens
|
||||
● shell(zizmor .github/workflows/ --format json --persona regular)
|
||||
✓ shell(zizmor .github/workflows/ --format json --persona regular)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 40348 in / 78 out tokens
|
||||
● echo(zizmor successfully ran and found security issues....)
|
||||
✓ echo(zizmor successfully ran and found security issues....)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 40414 in / 553 out tokens
|
||||
… Done
|
||||
|
||||
134
tests/fixtures/llm_traces/live/zizmor_scan_v2.json
vendored
134
tests/fixtures/llm_traces/live/zizmor_scan_v2.json
vendored
File diff suppressed because one or more lines are too long
@@ -1,28 +1,79 @@
|
||||
# Live Test Session: zizmor_scan_v2
|
||||
# Mode: Live
|
||||
# LLM calls: 4, Input tokens: 52200, Output tokens: 477
|
||||
# Wall time: 9.7s, Cost: $0.0224
|
||||
# LLM calls: 4, Input tokens: 100215, Output tokens: 1026
|
||||
# Wall time: 18.3s, Cost: $0.0000
|
||||
# ──────────────────────────────────────────────────
|
||||
|
||||
› can we run https://github.com/zizmorcore/zizmor
|
||||
## Turn 1
|
||||
[USER] › can we run https://github.com/zizmorcore/zizmor
|
||||
────────────────────────────────────────────────────
|
||||
|
||||
|
||||
Yes! Zizmor ran successfully and found **many security issues** in your GitHub Actions workflows. Here's a summary of the findings:
|
||||
|
||||
## 🔴 Critical Security Issues Found
|
||||
|
||||
### **Errors (High Severity):**
|
||||
1. **Excessive Permissions** - Multiple workflows have overly broad permissions:
|
||||
- `claude-review.yml`: pull-requests: write, issues: write, id-token: write
|
||||
- `pr-label-classify.yml`: pull-requests: write
|
||||
- `pr-label-scope.yml`: issues: write, pull-requests: write
|
||||
- `release-plz-batch-summary.yml`: pull-requests: write
|
||||
|
||||
2. **Dangerous Triggers** - `pull_request_target` used insecurely:
|
||||
- `pr-label-classify.yml`
|
||||
- `pr-label-scope.yml`
|
||||
- `release-plz-batch-summary.yml`
|
||||
- `staging-promotion-metadata.yml`
|
||||
|
||||
3. **Template Injection** - Code injection vulnerabilities:
|
||||
- Multiple workflows expose `${{ github.* }}` and `${{ inputs.* }}` in run blocks
|
||||
- Affected: `code_style.yml`, `rebuild-release-image.yml`, `staging-ci.yml`, `test.yml`
|
||||
|
||||
4. **Cache Poisoning** - `release.yml` uses rust-cache without `save-always: false`
|
||||
|
||||
5. **Unpinned Actions** - `replay-gate.yml` uses unpinned action references (@v4, @stable, @v2)
|
||||
|
||||
6. **Unpinned Images** - `coverage.yml` uses `pgvector/pgvector:pg16` without SHA hash
|
||||
|
||||
### **Warnings (Medium Severity):**
|
||||
- **Dependabot cooldown** - Missing cooldown configuration in `dependabot.yml`
|
||||
- **Artipacked** - `actions/checkout` without `persist-credentials: false` in multiple workflows
|
||||
- **Secrets inherit** - `release.yml` uses `secrets: inherit` when calling reusable workflows
|
||||
|
||||
### **Help/Info (Low Severity):**
|
||||
- **Undocumented permissions** - Many workflows lack explanatory comments for permissions
|
||||
- **Missing concurrency limits** - Most workflows lack `concurrency` settings
|
||||
- **Superfluous actions** - Using `dtolnay/rust-toolchain` when rustup/cargo is available
|
||||
|
||||
## 📊 Statistics:
|
||||
- **Total findings**: 100+ issues across 18 workflow files
|
||||
- **Errors**: ~15-20 critical issues
|
||||
- **Warnings**: ~10-15 medium issues
|
||||
- **Help/Info**: 70+ low-priority issues
|
||||
|
||||
## ✅ Auto-fixes Available:
|
||||
Many findings have auto-fixes that can be applied with `zizmor --fix`
|
||||
|
||||
Would you like me to:
|
||||
1. Generate a detailed JSON report of all findings?
|
||||
2. Apply auto-fixes to safe issues?
|
||||
3. Focus on specific high-priority issues?
|
||||
|
||||
## Activity
|
||||
○ Processing...
|
||||
○ Calling LLM...
|
||||
○ Step complete — 9624 in / 118 out tokens
|
||||
● tool_search(zizmor)
|
||||
✓ tool_search(zizmor)
|
||||
● http(https://github.com/zizmorcore/zizmor)
|
||||
✓ http(https://github.com/zizmorcore/zizmor)
|
||||
○ Step complete — 19610 in / 148 out tokens
|
||||
● shell(which zizmor || cargo install zizmor 2>&1 | head -20)
|
||||
✓ shell(which zizmor || cargo install zizmor 2>&1 | head -20)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 10895 in / 125 out tokens
|
||||
● tool_search(zizmor github actions security)
|
||||
✓ tool_search(zizmor github actions security)
|
||||
● tool_list
|
||||
✓ tool_list
|
||||
○ Step complete — 19742 in / 112 out tokens
|
||||
● shell(zizmor --help)
|
||||
✓ shell(zizmor --help)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 15823 in / 65 out tokens
|
||||
● tool_search(zizmor)
|
||||
✓ tool_search(zizmor)
|
||||
○ Step complete — 21386 in / 110 out tokens
|
||||
● shell(zizmor . --format plain --pedantic 2>&1)
|
||||
✓ shell(zizmor . --format plain --pedantic 2>&1)
|
||||
○ Calling LLM...
|
||||
○ Step complete — 15858 in / 169 out tokens
|
||||
────────────────────────────────────────────────────
|
||||
Tool 'shell' requires approval. Reply 'yes' to approve, 'no' to deny.
|
||||
○ Step complete — 39477 in / 656 out tokens
|
||||
… Done
|
||||
|
||||
Reference in New Issue
Block a user