//! Live integration test for the GitHub developer workflow. //! //! This is a **fully real** end-to-end test that exercises the //! `developer-setup` + `github-workflow` skills against the real //! `nearai/ironclaw` GitHub repo. The intent (per project owner) is to //! validate the workflow by doing useful work on the real repo and //! recording every interaction so we can debug what doesn't work and //! iterate on the skills. //! //! ## Flow //! //! 1. **Setup turn** — agent installs the workflow missions //! (`wf-issue-plan-*`, `wf-maintainer-gate-*`, `wf-pr-monitor-*`, //! `wf-ci-fix-*`, `wf-learning-*`) for `nearai/ironclaw` via real //! `mission_create` calls. //! //! 2. **Real issue creation** — the test (NOT the agent) opens a real //! issue on `nearai/ironclaw` via direct REST API. Title is prefixed //! `[live-test {timestamp}]` so it's identifiable. Issue URL is //! printed at the start so the human running the test can monitor //! or intervene. //! //! 3. **Triage turn** — the test tells the agent "issue #N just opened, //! please triage and post a plan." The agent reads the real issue, //! generates a plan, and posts a real comment back via the github //! skill. //! //! 4. **Verification** — the test polls the real issue's comments via //! REST and asserts that at least one new comment exists since the //! test started. Comment content is logged to the session log for //! human review (we don't assert on text since LLM output varies). //! //! 5. **Cleanup** — the test closes the real issue with a final //! "live-test complete" comment, regardless of pass/fail. If the //! test panics before reaching cleanup, the issue URL is in stderr //! so it can be closed manually. //! //! ## Why "real" instead of synthetic? //! //! An earlier version of this test used synthetic webhook payloads //! injected as channel messages. That approach kept the test hermetic //! but couldn't surface the realistic failure modes (auth gates, //! rate limits, payload format mismatches) that show up in production. //! Per project owner's direction, this version goes all-in on real //! artifacts so the recorded trace becomes authoritative debug data. //! //! The mission `OnSystemEvent` firing path (real webhook → mission → //! spawned thread) is NOT exercised here — that requires running an //! HTTP server and registering a real GitHub webhook, which is out of //! scope for this test. We exercise the **skill behavior** by driving //! the same agent conversation that a mission thread would drive. //! //! ## Running //! //! **Live mode** (default for this test — there is no replay yet): //! ```bash //! IRONCLAW_LIVE_TEST=1 cargo test --features libsql \ //! --test e2e_github_dev_workflow \ //! -- --ignored --test-threads=1 --nocapture //! ``` //! //! Requires: //! - `~/.ironclaw/.env` with valid LLM credentials //! - A `github_token` secret in `~/.ironclaw/ironclaw.db` with `repo` //! scope (the test rig copies it via `with_secrets(["github_token"])`) //! //! Replay mode is supported but requires the trace fixture to exist. //! On the first run with a fresh `.env` the test records the fixture. #[cfg(feature = "libsql")] mod support; #[cfg(feature = "libsql")] mod github_dev_workflow_test { use std::path::PathBuf; use std::time::Duration; use crate::support::live_harness::{LiveTestHarness, LiveTestHarnessBuilder, SessionTurn}; /// Repository under test. Owned and watched by the project owner; /// safe to create live-test issues against. const REPO_OWNER: &str = "nearai"; const REPO_NAME: &str = "ironclaw"; fn repo_skills_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("skills") } fn trace_fixture_path(test_name: &str) -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests") .join("fixtures") .join("llm_traces") .join("live") .join(format!("{test_name}.json")) } /// Skip in replay mode if the fixture doesn't exist yet. fn should_run_test(test_name: &str) -> bool { if trace_fixture_path(test_name).exists() || std::env::var("IRONCLAW_LIVE_TEST") .ok() .filter(|v| !v.is_empty() && v != "0") .is_some() { true } else { eprintln!( "[{}] replay fixture missing at {}; skipping until recorded in live mode", test_name, trace_fixture_path(test_name).display() ); false } } async fn build_workflow_harness(test_name: &str) -> LiveTestHarness { LiveTestHarnessBuilder::new(test_name) .with_engine_v2(true) .with_auto_approve_tools(true) // Workflow setup involves many sequential mission_create calls // plus per-event reasoning, so we need a generous iteration cap. .with_max_tool_iterations(80) .with_skills_dir(repo_skills_dir()) // Copy the real github_token from ~/.ironclaw/ironclaw.db so // the agent can talk to api.github.com. Required: the test // creates a real issue and the agent reads it + comments on it. .with_secrets(["github_token"]) .build() .await } /// Send a message and wait for at least `expected_responses` text replies. async fn run_turn( harness: &LiveTestHarness, message: &str, expected_responses: usize, ) -> Vec { let rig = harness.rig(); let before = rig.captured_responses().await.len(); rig.send_message(message).await; let responses = rig .wait_for_responses(before + expected_responses, Duration::from_secs(300)) .await; let new_responses: Vec = responses .into_iter() .skip(before) .map(|r| r.content) .collect(); assert!( !new_responses.is_empty(), "Expected at least one response to: {message}" ); new_responses } /// Dump captured tool activity to stderr after each turn so failing /// runs surface what the agent actually did. fn dump_activity(harness: &LiveTestHarness, label: &str) { use ironclaw::channels::StatusUpdate; eprintln!("───── [{label}] activity dump ─────"); eprintln!("active skills: {:?}", harness.rig().active_skill_names()); for event in harness.rig().captured_status_events() { match event { StatusUpdate::SkillActivated { skill_names } => { eprintln!(" ◆ skills activated: {}", skill_names.join(", ")); } StatusUpdate::ToolStarted { name, detail, .. } => { eprintln!(" ● {name} {}", detail.unwrap_or_default()); } StatusUpdate::ToolCompleted { name, success, error, .. } => { if success { eprintln!(" ✓ {name}"); } else { eprintln!(" ✗ {name}: {}", error.unwrap_or_default()); } } StatusUpdate::ToolResult { name, preview, .. } => { let short: String = preview.chars().take(200).collect(); eprintln!(" {name} → {short}"); } _ => {} } } eprintln!("───── end activity ─────"); } // ───────────────────────────────────────────────────────────────────── // Direct GitHub REST helpers // // These run inside the test process (not via the agent) so the test // can set up real artifacts before the agent runs and verify/clean up // afterwards. They use the same `github_token` the agent uses (read // back from the rig's SecretsStore via `rig.get_secret`). // // We use reqwest directly rather than the agent's `http` tool because // the test needs guaranteed access to GitHub regardless of skill // selection / tool gating. // ───────────────────────────────────────────────────────────────────── mod github_api { use serde_json::Value; const GITHUB_API: &str = "https://api.github.com"; fn client() -> reqwest::Client { reqwest::Client::builder() .user_agent("ironclaw-live-test/0.1") .build() .expect("build reqwest client") } /// Open a real issue. Returns `(issue_number, html_url)`. pub async fn create_issue( token: &str, owner: &str, repo: &str, title: &str, body: &str, ) -> Result<(u64, String), String> { let url = format!("{GITHUB_API}/repos/{owner}/{repo}/issues"); let resp = client() .post(&url) .header("Accept", "application/vnd.github+json") .header("X-GitHub-Api-Version", "2022-11-28") .bearer_auth(token) .json(&serde_json::json!({ "title": title, "body": body, "labels": ["live-test"], })) .send() .await .map_err(|e| format!("create_issue request: {e}"))?; let status = resp.status(); let body_text = resp .text() .await .map_err(|e| format!("create_issue body: {e}"))?; if !status.is_success() { return Err(format!("create_issue {status}: {body_text}")); } let v: Value = serde_json::from_str(&body_text).map_err(|e| format!("create_issue parse: {e}"))?; let number = v .get("number") .and_then(|n| n.as_u64()) .ok_or_else(|| format!("create_issue: no number in response: {body_text}"))?; let html_url = v .get("html_url") .and_then(|s| s.as_str()) .map(String::from) .unwrap_or_else(|| format!("https://github.com/{owner}/{repo}/issues/{number}")); Ok((number, html_url)) } /// List comments on an issue. Returns the raw JSON array. pub async fn list_issue_comments( token: &str, owner: &str, repo: &str, issue_number: u64, ) -> Result, String> { let url = format!( "{GITHUB_API}/repos/{owner}/{repo}/issues/{issue_number}/comments?per_page=100" ); let resp = client() .get(&url) .header("Accept", "application/vnd.github+json") .header("X-GitHub-Api-Version", "2022-11-28") .bearer_auth(token) .send() .await .map_err(|e| format!("list_issue_comments request: {e}"))?; let status = resp.status(); let body_text = resp .text() .await .map_err(|e| format!("list_issue_comments body: {e}"))?; if !status.is_success() { return Err(format!("list_issue_comments {status}: {body_text}")); } let v: Value = serde_json::from_str(&body_text) .map_err(|e| format!("list_issue_comments parse: {e}"))?; Ok(v.as_array().cloned().unwrap_or_default()) } /// Post a comment on an issue (used by the test for the LGTM /// confirmation step and for the final cleanup notice). pub async fn post_issue_comment( token: &str, owner: &str, repo: &str, issue_number: u64, body: &str, ) -> Result<(), String> { let url = format!("{GITHUB_API}/repos/{owner}/{repo}/issues/{issue_number}/comments"); let resp = client() .post(&url) .header("Accept", "application/vnd.github+json") .header("X-GitHub-Api-Version", "2022-11-28") .bearer_auth(token) .json(&serde_json::json!({ "body": body })) .send() .await .map_err(|e| format!("post_issue_comment request: {e}"))?; let status = resp.status(); if !status.is_success() { let body_text = resp.text().await.unwrap_or_default(); return Err(format!("post_issue_comment {status}: {body_text}")); } Ok(()) } /// Close an issue. Used by cleanup at the end of the test. pub async fn close_issue( token: &str, owner: &str, repo: &str, issue_number: u64, ) -> Result<(), String> { let url = format!("{GITHUB_API}/repos/{owner}/{repo}/issues/{issue_number}"); let resp = client() .patch(&url) .header("Accept", "application/vnd.github+json") .header("X-GitHub-Api-Version", "2022-11-28") .bearer_auth(token) .json(&serde_json::json!({ "state": "closed" })) .send() .await .map_err(|e| format!("close_issue request: {e}"))?; let status = resp.status(); if !status.is_success() { let body_text = resp.text().await.unwrap_or_default(); return Err(format!("close_issue {status}: {body_text}")); } Ok(()) } } /// Best-effort cleanup helper. Posts a final notice and closes the /// issue. Logs failures to stderr instead of panicking — cleanup /// should never mask the original test result. async fn cleanup_issue(token: &str, issue_number: u64) { let final_comment = "🤖 **Live test complete.** Closing this issue.\n\n\ This issue was created by the IronClaw `e2e_github_dev_workflow` \ live integration test. If you're seeing this and the test was \ still useful, the recorded trace is at \ `tests/fixtures/llm_traces/live/github_dev_workflow_full_loop.json`."; if let Err(e) = github_api::post_issue_comment( token, REPO_OWNER, REPO_NAME, issue_number, final_comment, ) .await { eprintln!("[cleanup] WARNING: failed to post final comment: {e}"); } if let Err(e) = github_api::close_issue(token, REPO_OWNER, REPO_NAME, issue_number).await { eprintln!("[cleanup] WARNING: failed to close issue #{issue_number}: {e}"); eprintln!( "[cleanup] Manual cleanup needed: \ https://github.com/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}" ); } else { eprintln!("[cleanup] closed issue #{issue_number}"); } } #[tokio::test] #[ignore] // Live tier: requires LLM API keys + github_token in ~/.ironclaw async fn github_dev_workflow_full_loop() { let test_name = "github_dev_workflow_full_loop"; if !should_run_test(test_name) { return; } let harness = build_workflow_harness(test_name).await; let mut transcript: Vec = Vec::new(); // Pull the github token back out of the rig's secrets store so // we can issue direct REST calls. The harness already attempted // to seed it via with_secrets(["github_token"]) during build. // // This test is **inherently live-only** for the GitHub side: it // creates real issues, polls real comments, and closes real // issues regardless of whether the LLM is replayed from a // fixture. If the token is missing we skip gracefully — there // is no useful pure-replay mode for a test whose verification // step is "did a real comment appear on a real GitHub issue". let Some(github_token) = harness.rig().get_secret("github_token").await else { eprintln!( "[{test_name}] github_token not found in ~/.ironclaw/ironclaw.db; \ skipping. This test makes real GitHub API calls and cannot run \ in pure replay mode. To enable: configure a github_token secret \ in your local ironclaw setup and rerun with IRONCLAW_LIVE_TEST=1." ); return; }; // ── Turn 1: Setup workflow ─────────────────────────────────── // The agent installs the wf-* mission set for nearai/ironclaw. // We don't strictly need this turn for the triage flow below, // but it exercises the github-workflow skill's install path // which is the other half of the dev workflow surface area. let setup_msg = "I'm a software engineer. Set up the GitHub workflow for \ nearai/ironclaw. Maintainers: ilblackdragon. Staging \ branch: staging. Do NOT install the staging-batch-review \ mission — humans will merge to main. Use sensible defaults \ and skip the setup questions."; let setup_responses = run_turn(&harness, setup_msg, 1).await; eprintln!("[setup] response: {}", setup_responses.join("\n")); dump_activity(&harness, "after setup"); // Setup must have called mission_create at least once and the // response must reference the wf-* templates. harness.assert_trace_contains_tool_call( "mission_create", "", "Setup turn: at least one mission_create call required", ); let setup_text = setup_responses.join("\n").to_lowercase(); assert!( setup_text.contains("wf-issue-plan") || setup_text.contains("wf-pr-monitor") || setup_text.contains("wf-ci-fix"), "Setup turn: response should mention at least one wf-* mission name. Got: {}", setup_text.chars().take(400).collect::(), ); transcript.push(SessionTurn::user(setup_msg, setup_responses)); // ── Create a real test issue on nearai/ironclaw ────────────── let timestamp = chrono::Utc::now().format("%Y-%m-%d %H:%M UTC"); let issue_title = format!("[live-test {timestamp}] Add /metrics Prometheus endpoint"); let issue_body = "**This is an automated live integration test issue.** It was opened by \ `tests/e2e_github_dev_workflow.rs::github_dev_workflow_full_loop` to exercise the \ `developer-setup` + `github-workflow` skills end-to-end against a real \ repository.\n\n\ ## Feature request\n\n\ Expose Prometheus-style metrics for IronClaw at `/metrics`. Should include:\n\n\ - Request latency histograms per channel + per tool\n\ - Tool execution count + success rate\n\ - Active session count + thread count\n\ - LLM token usage counters per model + per backend\n\n\ The endpoint should not require auth in single-user mode (the typical local \ ironclaw deployment); for multi-user gateway deployments it should require the \ existing admin credential.\n\n\ ## Why this matters\n\n\ Production observability is currently limited to `tracing` log output. A \ scrapeable metrics endpoint unlocks dashboards, alerting, and SLO tracking \ without log-aggregation pipelines.\n\n\ ---\n\n\ 🤖 The agent will respond to this issue with a triage and an implementation plan. \ **The test will close this issue automatically after recording the agent's response.** \ If you're a human reading this and the issue is still open after a few minutes, the \ test panicked — see the test output for the last activity dump."; let (issue_number, issue_url) = github_api::create_issue( &github_token, REPO_OWNER, REPO_NAME, &issue_title, issue_body, ) .await .expect("create real test issue on nearai/ironclaw"); eprintln!("[live-test] created real issue #{issue_number}: {issue_url}"); // Wrap everything after issue creation in a guard so cleanup // runs even if an assertion or .expect() panics. This prevents // orphaned issues on the real repo. let test_result = std::panic::AssertUnwindSafe(async { // Capture the comment count baseline so we can detect new // comments posted by the agent. let baseline_comments = github_api::list_issue_comments(&github_token, REPO_OWNER, REPO_NAME, issue_number) .await .expect("baseline list_issue_comments") .len(); eprintln!("[live-test] baseline comment count: {baseline_comments}"); // ── Turn 2: Triage the real issue ──────────────────────── let triage_msg = format!( "Issue #{issue_number} just opened on {REPO_OWNER}/{REPO_NAME}: \ \"{issue_title}\". Please read the issue, triage it, and post a \ comment with a concrete implementation plan. The plan should \ include: scope (what's in/out), milestones, risks, and a \ testing strategy. Use the github skill to read the issue body \ and to post your plan as an issue comment.", ); let triage_responses = run_turn(&harness, &triage_msg, 1).await; eprintln!("[triage] response: {}", triage_responses.join("\n")); dump_activity(&harness, "after triage"); transcript.push(SessionTurn::user(&triage_msg, triage_responses)); // ── Verification: poll the real issue for new comments ── // The github API can be eventually-consistent on read-after- // write, so we poll for up to 30s. let deadline = std::time::Instant::now() + Duration::from_secs(30); let latest_comments; loop { let snapshot = github_api::list_issue_comments( &github_token, REPO_OWNER, REPO_NAME, issue_number, ) .await .expect("list_issue_comments after triage"); if snapshot.len() > baseline_comments { latest_comments = snapshot; break; } if std::time::Instant::now() >= deadline { latest_comments = snapshot; break; } tokio::time::sleep(Duration::from_secs(2)).await; } assert!( latest_comments.len() > baseline_comments, "Triage turn: expected agent to post at least one new comment on \ issue #{issue_number} (baseline {baseline_comments}, current {}). \ The agent may have replied conversationally instead of using \ the github skill — see the activity dump above for actual \ tool calls. Issue: {issue_url}", latest_comments.len(), ); // Log new comments to stderr so the human can review what // the agent actually wrote — this is the most useful output // of the test for iterating on skill quality. let new_count = latest_comments.len() - baseline_comments; eprintln!( "[live-test] ✅ agent posted {new_count} new comment(s) on issue #{issue_number}" ); for (i, comment) in latest_comments.iter().skip(baseline_comments).enumerate() { let author = comment .get("user") .and_then(|u| u.get("login")) .and_then(|s| s.as_str()) .unwrap_or("?"); let body = comment .get("body") .and_then(|s| s.as_str()) .unwrap_or("(empty)"); let body_preview: String = body.chars().take(800).collect(); eprintln!( "[live-test] new comment {} by @{author}:\n{body_preview}\n", i + 1 ); } }); let test_outcome = futures::FutureExt::catch_unwind(test_result).await; // ── Cleanup: always close the issue ────────────────────────── cleanup_issue(&github_token, issue_number).await; // Re-raise any panic so cargo test sees the failure. if let Err(panic_payload) = test_outcome { std::panic::resume_unwind(panic_payload); } // ── Final: workflow + github skills must have activated ────── let active = harness.rig().active_skill_names(); for required in ["github-workflow", "github"] { assert!( active.iter().any(|s| s == required), "Expected skill '{required}' to activate during {test_name}. Active: {active:?}", ); } harness.finish_turns_strict(&transcript).await; } }