From 7db0df6b0aa7a736bc542e2f81ef8777e9f1e351 Mon Sep 17 00:00:00 2001 From: "firat.sertgoz" Date: Thu, 7 May 2026 10:55:55 +0300 Subject: [PATCH] test(reborn): add dedicated e2e gate (#3251) --- .github/workflows/reborn-e2e.yml | 140 ++++ .../tests/reborn_e2e_gate.rs | 735 ++++++++++++++++++ docs/reborn/harness/e2e.md | 146 ++++ scripts/reborn-e2e-rust.sh | 104 +++ .../scenarios/test_reborn_gateway_smoke.py | 265 +++++++ 5 files changed, 1390 insertions(+) create mode 100644 .github/workflows/reborn-e2e.yml create mode 100644 crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs create mode 100644 docs/reborn/harness/e2e.md create mode 100755 scripts/reborn-e2e-rust.sh create mode 100644 tests/e2e/scenarios/test_reborn_gateway_smoke.py diff --git a/.github/workflows/reborn-e2e.yml b/.github/workflows/reborn-e2e.yml new file mode 100644 index 0000000000..a06497f82f --- /dev/null +++ b/.github/workflows/reborn-e2e.yml @@ -0,0 +1,140 @@ +name: Reborn E2E + +on: + workflow_call: + inputs: + ref: + description: Commit SHA or ref to test + required: false + type: string + workflow_dispatch: + pull_request: + branches: + - main + paths: + - "crates/ironclaw_*/**" + - "docs/reborn/**" + - "scripts/reborn-e2e-rust.sh" + - "tests/e2e/**" + - "build.rs" + - "providers.json" + - "Cargo.toml" + - "Cargo.lock" + - ".github/workflows/reborn-e2e.yml" + push: + branches: + - main + paths: + - "crates/ironclaw_*/**" + - "docs/reborn/**" + - "scripts/reborn-e2e-rust.sh" + - "tests/e2e/**" + - "build.rs" + - "providers.json" + - "Cargo.toml" + - "Cargo.lock" + - ".github/workflows/reborn-e2e.yml" + +permissions: + contents: read + +concurrency: + group: reborn-e2e-${{ github.event_name }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +jobs: + rust-reborn: + name: Rust Reborn (${{ matrix.group }}) + runs-on: ubuntu-latest + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + group: + - architecture + - runtimes + - substrates + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + - name: Restore Rust cache + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + key: reborn-e2e-${{ matrix.group }} + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Run deterministic Reborn Rust gate + env: + CARGO_TEST_ARGS: "-- --nocapture" + run: scripts/reborn-e2e-rust.sh ${{ matrix.group }} + + gateway-smoke: + name: Reborn gateway smoke + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + ref: ${{ inputs.ref || github.sha }} + persist-credentials: false + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + - name: Restore Rust cache + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + key: reborn-e2e-gateway-smoke + save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + + - name: Build ironclaw with libSQL + run: cargo build --no-default-features --features libsql + + - name: Install Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + + - name: Install E2E dependencies + run: | + cd tests/e2e + pip install -e . + playwright install --with-deps chromium + + - name: Run Reborn gateway smoke + run: pytest tests/e2e/scenarios/test_reborn_gateway_smoke.py -v --timeout=120 + + - name: Upload screenshots on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: reborn-e2e-screenshots + path: tests/e2e/screenshots/ + if-no-files-found: ignore + + reborn-e2e: + name: Reborn E2E + runs-on: ubuntu-latest + if: always() + needs: + - rust-reborn + - gateway-smoke + steps: + - name: Check Reborn E2E jobs + run: | + if [[ "${{ needs.rust-reborn.result }}" != "success" ]]; then + echo "Rust Reborn E2E failed" + exit 1 + fi + if [[ "${{ needs.gateway-smoke.result }}" != "success" ]]; then + echo "Reborn gateway smoke failed" + exit 1 + fi diff --git a/crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs b/crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs new file mode 100644 index 0000000000..8d8c28e678 --- /dev/null +++ b/crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs @@ -0,0 +1,735 @@ +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use chrono::Utc; +use ironclaw_approvals::LeaseApproval; +use ironclaw_authorization::{ + CapabilityLeaseStatus, CapabilityLeaseStore, GrantAuthorizer, InMemoryCapabilityLeaseStore, + TrustAwareCapabilityDispatchAuthorizer, +}; +use ironclaw_events::{ + DurableEventLog, EventStreamKey, InMemoryDurableEventLog, InMemoryEventSink, ReadScope, + RuntimeEventKind, +}; +use ironclaw_extensions::{ExtensionManifest, ExtensionPackage, ExtensionRegistry}; +use ironclaw_filesystem::LocalFilesystem; +use ironclaw_host_api::*; +use ironclaw_host_runtime::{ + CapabilitySurfaceVersion, HostHttpEgressService, HostRuntime, HostRuntimeServices, + NetworkObligationPolicyStore, RuntimeCapabilityOutcome, RuntimeCapabilityRequest, + RuntimeCapabilityResumeRequest, RuntimeFailureKind, RuntimeSecretInjectionStore, + RuntimeStatusRequest, SurfaceKind, +}; +use ironclaw_network::{ + NetworkHttpEgress, NetworkHttpError, NetworkHttpRequest, NetworkHttpResponse, NetworkUsage, +}; +use ironclaw_processes::{InMemoryProcessResultStore, InMemoryProcessStore, ProcessServices}; +use ironclaw_resources::{InMemoryResourceGovernor, ResourceAccount, ResourceTally}; +use ironclaw_run_state::{ + InMemoryApprovalRequestStore, InMemoryRunStateStore, RunStateStore, RunStatus, +}; +use ironclaw_scripts::{ + ScriptBackend, ScriptBackendOutput, ScriptBackendRequest, ScriptRuntime, ScriptRuntimeConfig, +}; +use ironclaw_secrets::{InMemorySecretStore, SecretMaterial}; +use ironclaw_trust::{ + AdminConfig, AdminEntry, AuthorityCeiling, EffectiveTrustClass, HostTrustAssignment, + HostTrustPolicy, TrustDecision, TrustProvenance, +}; +use serde_json::json; + +#[tokio::test] +async fn reborn_e2e_gate_invokes_script_through_host_runtime_with_status_events_and_resources() { + let governor = Arc::new(InMemoryResourceGovernor::new()); + let run_state = Arc::new(InMemoryRunStateStore::new()); + let event_log = Arc::new(InMemoryDurableEventLog::new()); + let services = HostRuntimeServices::new( + Arc::new(registry_with_manifest(SCRIPT_MANIFEST)), + Arc::new(LocalFilesystem::new()), + Arc::clone(&governor), + Arc::new(GrantAuthorizer::new()), + ProcessServices::in_memory(), + CapabilitySurfaceVersion::new("surface-v1").unwrap(), + ) + .with_trust_policy(Arc::new(local_manifest_trust_policy())) + .with_run_state(Arc::clone(&run_state)) + .with_script_runtime(Arc::new(ScriptRuntime::new( + ScriptRuntimeConfig::for_testing(), + EchoScriptBackend, + ))) + .with_durable_event_log(Arc::clone(&event_log)); + let runtime = services.host_runtime(); + let context = execution_context_with_dispatch_grant(); + let scope = context.resource_scope.clone(); + let invocation_id = context.invocation_id; + + let surface = runtime + .visible_capabilities(ironclaw_host_runtime::VisibleCapabilityRequest::new( + scope.clone(), + context.correlation_id, + SurfaceKind::new("gateway-smoke").unwrap(), + )) + .await + .unwrap(); + assert_eq!(surface.version.as_str(), "surface-v1"); + assert_eq!(surface.descriptors.len(), 1); + assert_eq!(surface.descriptors[0].id, script_capability_id()); + + let health = runtime.health().await.unwrap(); + assert!(health.ready); + assert!(health.missing_runtime_backends.is_empty()); + + let status_before = runtime + .runtime_status(RuntimeStatusRequest::new( + scope.clone(), + CorrelationId::new(), + )) + .await + .unwrap(); + assert!(status_before.active_work.is_empty()); + + let input = json!({ + "message": "reborn e2e happy path", + "secret_sentinel": "SECRET_REBORN_E2E_GATE_SHOULD_NOT_LEAK", + "host_path_sentinel": "/private/tmp/reborn-e2e-gate" + }); + let outcome = runtime + .invoke_capability(RuntimeCapabilityRequest::new( + context, + script_capability_id(), + ResourceEstimate { + output_bytes: Some(4096), + ..ResourceEstimate::default() + }, + input.clone(), + trust_decision_with_dispatch_authority(), + )) + .await + .unwrap(); + + match outcome { + RuntimeCapabilityOutcome::Completed(completed) => { + assert_eq!(completed.capability_id, script_capability_id()); + assert_eq!(completed.output, input); + assert!(completed.usage.output_bytes > 0); + } + other => panic!("expected completed outcome, got {other:?}"), + } + + let run = run_state.get(&scope, invocation_id).await.unwrap().unwrap(); + assert_eq!(run.status, RunStatus::Completed); + let tenant_account = ResourceAccount::tenant(scope.tenant_id.clone()); + assert_eq!( + governor.reserved_for(&tenant_account), + ResourceTally::default() + ); + assert!(governor.usage_for(&tenant_account).output_bytes > 0); + + let status_after = runtime + .runtime_status(RuntimeStatusRequest::new( + scope.clone(), + CorrelationId::new(), + )) + .await + .unwrap(); + assert!(status_after.active_work.is_empty()); + + let replay = event_log + .read_after_cursor( + &EventStreamKey::from_scope(&scope), + &ReadScope::any(), + None, + 10, + ) + .await + .unwrap(); + let kinds = replay + .entries + .iter() + .map(|entry| entry.record.kind) + .collect::>(); + assert_eq!( + kinds, + vec![ + RuntimeEventKind::DispatchRequested, + RuntimeEventKind::RuntimeSelected, + RuntimeEventKind::DispatchSucceeded, + ] + ); + let serialized = serde_json::to_string(&replay).unwrap(); + for forbidden in [ + "SECRET_REBORN_E2E_GATE_SHOULD_NOT_LEAK", + "/private/tmp/reborn-e2e-gate", + ] { + assert!( + !serialized.contains(forbidden), + "durable Reborn E2E event replay leaked {forbidden}: {serialized}" + ); + } +} + +#[tokio::test] +async fn reborn_e2e_gate_blocks_for_approval_resumes_once_and_rejects_replay() { + let fixture = approval_resume_fixture(); + let runtime = fixture.services.host_runtime(); + let context = execution_context_without_grants(); + let scope = context.resource_scope.clone(); + let invocation_id = context.invocation_id; + let input = json!({"message": "approval resume through Reborn E2E gate"}); + + let gate = block_for_approval(&runtime, context.clone(), input.clone()).await; + let blocked_run = fixture + .run_state + .get(&scope, invocation_id) + .await + .unwrap() + .unwrap(); + assert_eq!(blocked_run.status, RunStatus::BlockedApproval); + assert_eq!( + blocked_run.approval_request_id, + Some(gate.approval_request_id) + ); + + let lease = + approve_dispatch_for_services(&fixture.services, &scope, gate.approval_request_id).await; + + let resumed = runtime + .resume_capability(RuntimeCapabilityResumeRequest::new( + context.clone(), + gate.approval_request_id, + script_capability_id(), + ResourceEstimate::default(), + input.clone(), + trust_decision_with_dispatch_authority(), + )) + .await + .unwrap(); + match resumed { + RuntimeCapabilityOutcome::Completed(completed) => { + assert_eq!(completed.capability_id, script_capability_id()); + assert_eq!(completed.output, input); + } + other => panic!("expected completed approval resume, got {other:?}"), + } + assert_eq!( + fixture + .capability_leases + .get(&scope, lease.grant.id) + .await + .unwrap() + .status, + CapabilityLeaseStatus::Consumed + ); + assert_event_kinds( + &fixture.events, + &[ + RuntimeEventKind::DispatchRequested, + RuntimeEventKind::RuntimeSelected, + RuntimeEventKind::DispatchSucceeded, + ], + ); + + let replay = runtime + .resume_capability(RuntimeCapabilityResumeRequest::new( + context, + gate.approval_request_id, + script_capability_id(), + ResourceEstimate::default(), + json!({"message": "approval resume through Reborn E2E gate"}), + trust_decision_with_dispatch_authority(), + )) + .await + .unwrap(); + assert_failed_outcome(replay, RuntimeFailureKind::Authorization); + assert_eq!( + fixture.events.events().len(), + 3, + "replayed approval resume must fail before a second runtime dispatch" + ); +} + +#[tokio::test] +async fn reborn_e2e_gate_fails_unsupported_obligations_before_runtime_events_or_success() { + let run_state = Arc::new(InMemoryRunStateStore::new()); + let events = InMemoryEventSink::new(); + let governor = Arc::new(InMemoryResourceGovernor::new()); + let services = HostRuntimeServices::new( + Arc::new(registry_with_manifest(SCRIPT_MANIFEST)), + Arc::new(LocalFilesystem::new()), + Arc::clone(&governor), + Arc::new(ObligatingAuthorizer), + ProcessServices::in_memory(), + CapabilitySurfaceVersion::new("surface-v1").unwrap(), + ) + .with_trust_policy(Arc::new(local_manifest_trust_policy())) + .with_run_state(Arc::clone(&run_state)) + .with_script_runtime(Arc::new(ScriptRuntime::new( + ScriptRuntimeConfig::for_testing(), + EchoScriptBackend, + ))) + .with_event_sink(Arc::new(events.clone())); + let runtime = services.host_runtime(); + let context = execution_context_with_dispatch_grant(); + let scope = context.resource_scope.clone(); + let invocation_id = context.invocation_id; + + let outcome = runtime + .invoke_capability(RuntimeCapabilityRequest::new( + context, + script_capability_id(), + ResourceEstimate::default(), + json!({"message": "unsupported obligation"}), + trust_decision_with_dispatch_authority(), + )) + .await + .unwrap(); + + assert_failed_outcome(outcome, RuntimeFailureKind::Backend); + assert!(events.events().is_empty()); + let run = run_state.get(&scope, invocation_id).await.unwrap().unwrap(); + assert_eq!(run.status, RunStatus::Failed); + assert_eq!(run.error_kind.as_deref(), Some("ObligationFailed")); + let tenant_account = ResourceAccount::tenant(scope.tenant_id.clone()); + assert_eq!( + governor.reserved_for(&tenant_account), + ResourceTally::default() + ); + assert_eq!( + governor.usage_for(&tenant_account), + ResourceTally::default() + ); +} + +#[test] +fn reborn_e2e_gate_host_http_consumes_staged_policy_and_secret_once() { + let network = RecordingNetwork::ok(NetworkHttpResponse { + status: 200, + headers: vec![], + body: br#"{"ok":true}"#.to_vec(), + usage: NetworkUsage { + request_bytes: 5, + response_bytes: 11, + resolved_ip: None, + }, + }); + let network_recorder = network.requests.clone(); + let policy_store = Arc::new(NetworkObligationPolicyStore::new()); + let secret_injections = Arc::new(RuntimeSecretInjectionStore::new()); + let scope = sample_scope(InvocationId::new()); + let capability_id = script_capability_id(); + let handle = SecretHandle::new("api-token").unwrap(); + let staged_policy = sample_policy(); + policy_store.insert(&scope, &capability_id, staged_policy.clone()); + secret_injections + .insert( + &scope, + &capability_id, + &handle, + SecretMaterial::from("sk-reborn-e2e-staged-secret"), + ) + .unwrap(); + let service = HostHttpEgressService::new(network, InMemorySecretStore::new()) + .with_network_policy_store(Arc::clone(&policy_store)) + .with_secret_injection_store(Arc::clone(&secret_injections)); + + let request = RuntimeHttpEgressRequest { + runtime: RuntimeKind::Script, + scope: scope.clone(), + capability_id: capability_id.clone(), + method: NetworkMethod::Post, + url: "https://api.example.test/v1/run".to_string(), + headers: vec![], + body: b"hello".to_vec(), + network_policy: caller_supplied_policy(), + credential_injections: vec![RuntimeCredentialInjection { + handle: handle.clone(), + source: RuntimeCredentialSource::StagedObligation { + capability_id: capability_id.clone(), + }, + target: RuntimeCredentialTarget::Header { + name: "authorization".to_string(), + prefix: Some("Bearer ".to_string()), + }, + required: true, + }], + response_body_limit: Some(4096), + timeout_ms: None, + }; + + let response = service + .execute(request.clone()) + .expect("host HTTP egress should use staged Reborn policy and secret material"); + assert_eq!(response.status, 200); + let recorded = network_recorder.lock().unwrap(); + assert_eq!(recorded.len(), 1); + assert_eq!(recorded[0].policy, staged_policy); + assert_eq!( + recorded[0] + .headers + .iter() + .find(|(name, _)| name == "authorization"), + Some(&( + "authorization".to_string(), + "Bearer sk-reborn-e2e-staged-secret".to_string() + )) + ); + drop(recorded); + assert!( + secret_injections + .take(&scope, &capability_id, &handle) + .unwrap() + .is_none(), + "staged secret material must be consumed exactly once" + ); + assert_eq!( + policy_store.get(&scope, &capability_id), + Some(staged_policy), + "host egress must leave staged network policy for invocation/process lifecycle cleanup" + ); + + let replay = service + .execute(request) + .expect_err("consumed staged secret must not be reusable"); + assert!(matches!(replay, RuntimeHttpEgressError::Credential { .. })); + assert_eq!( + network_recorder.lock().unwrap().len(), + 1, + "replay must fail before a second outbound transport attempt" + ); +} + +type InMemoryServices = HostRuntimeServices< + LocalFilesystem, + InMemoryResourceGovernor, + InMemoryProcessStore, + InMemoryProcessResultStore, +>; + +struct ApprovalFixture { + services: InMemoryServices, + run_state: Arc, + capability_leases: Arc, + events: InMemoryEventSink, +} + +fn approval_resume_fixture() -> ApprovalFixture { + let run_state = Arc::new(InMemoryRunStateStore::new()); + let approval_requests = Arc::new(InMemoryApprovalRequestStore::new()); + let capability_leases = Arc::new(InMemoryCapabilityLeaseStore::new()); + let events = InMemoryEventSink::new(); + let services = HostRuntimeServices::new( + Arc::new(registry_with_manifest(SCRIPT_MANIFEST)), + Arc::new(LocalFilesystem::new()), + Arc::new(InMemoryResourceGovernor::new()), + Arc::new(ApprovalThenGrantAuthorizer), + ProcessServices::in_memory(), + CapabilitySurfaceVersion::new("surface-v1").unwrap(), + ) + .with_trust_policy(Arc::new(local_manifest_trust_policy())) + .with_run_state(Arc::clone(&run_state)) + .with_approval_requests(approval_requests) + .with_capability_leases(Arc::clone(&capability_leases)) + .with_script_runtime(Arc::new(ScriptRuntime::new( + ScriptRuntimeConfig::for_testing(), + EchoScriptBackend, + ))) + .with_event_sink(Arc::new(events.clone())); + + ApprovalFixture { + services, + run_state, + capability_leases, + events, + } +} + +async fn block_for_approval( + runtime: &impl HostRuntime, + context: ExecutionContext, + input: serde_json::Value, +) -> ironclaw_host_runtime::RuntimeApprovalGate { + let outcome = runtime + .invoke_capability(RuntimeCapabilityRequest::new( + context, + script_capability_id(), + ResourceEstimate::default(), + input, + trust_decision_with_dispatch_authority(), + )) + .await + .unwrap(); + match outcome { + RuntimeCapabilityOutcome::ApprovalRequired(gate) => gate, + other => panic!("expected approval gate, got {other:?}"), + } +} + +async fn approve_dispatch_for_services( + services: &InMemoryServices, + scope: &ResourceScope, + approval_request_id: ApprovalRequestId, +) -> ironclaw_authorization::CapabilityLease { + services + .approval_resolver() + .expect("approval resolver should be configured") + .approve_dispatch( + scope, + approval_request_id, + LeaseApproval { + issued_by: Principal::HostRuntime, + allowed_effects: vec![EffectKind::DispatchCapability], + mounts: MountView::default(), + network: NetworkPolicy::default(), + secrets: Vec::new(), + resource_ceiling: None, + expires_at: None, + max_invocations: Some(1), + }, + ) + .await + .unwrap() +} + +struct ApprovalThenGrantAuthorizer; + +#[async_trait] +impl TrustAwareCapabilityDispatchAuthorizer for ApprovalThenGrantAuthorizer { + async fn authorize_dispatch_with_trust( + &self, + context: &ExecutionContext, + descriptor: &CapabilityDescriptor, + estimate: &ResourceEstimate, + trust_decision: &TrustDecision, + ) -> Decision { + if context.grants.grants.is_empty() { + Decision::RequireApproval { + request: ApprovalRequest { + id: ApprovalRequestId::new(), + correlation_id: context.correlation_id, + requested_by: Principal::Extension(context.extension_id.clone()), + action: Box::new(Action::Dispatch { + capability: descriptor.id.clone(), + estimated_resources: estimate.clone(), + }), + invocation_fingerprint: None, + reason: "approval required".to_string(), + reusable_scope: None, + }, + } + } else { + GrantAuthorizer::new() + .authorize_dispatch_with_trust(context, descriptor, estimate, trust_decision) + .await + } + } +} + +struct ObligatingAuthorizer; + +#[async_trait] +impl TrustAwareCapabilityDispatchAuthorizer for ObligatingAuthorizer { + async fn authorize_dispatch_with_trust( + &self, + _context: &ExecutionContext, + _descriptor: &CapabilityDescriptor, + _estimate: &ResourceEstimate, + _trust_decision: &TrustDecision, + ) -> Decision { + Decision::Allow { + obligations: Obligations::new(vec![Obligation::AuditBefore]).unwrap(), + } + } +} + +struct EchoScriptBackend; + +impl ScriptBackend for EchoScriptBackend { + fn execute(&self, request: ScriptBackendRequest) -> Result { + let value = serde_json::from_str(&request.stdin_json).map_err(|error| error.to_string())?; + Ok(ScriptBackendOutput::json(value)) + } +} + +#[derive(Clone)] +struct RecordingNetwork { + response: Result, + requests: Arc>>, +} + +impl RecordingNetwork { + fn ok(response: NetworkHttpResponse) -> Self { + Self { + response: Ok(response), + requests: Arc::new(Mutex::new(Vec::new())), + } + } +} + +impl NetworkHttpEgress for RecordingNetwork { + fn execute( + &self, + request: NetworkHttpRequest, + ) -> Result { + self.requests.lock().unwrap().push(request); + self.response.clone() + } +} + +fn registry_with_manifest(manifest: &str) -> ExtensionRegistry { + let mut registry = ExtensionRegistry::new(); + let manifest = ExtensionManifest::parse(manifest).unwrap(); + let package = ExtensionPackage::from_manifest( + manifest, + VirtualPath::new("/system/extensions/script").unwrap(), + ) + .unwrap(); + registry.insert(package).unwrap(); + registry +} + +fn execution_context_with_dispatch_grant() -> ExecutionContext { + let mut grants = CapabilitySet::default(); + grants.grants.push(dispatch_grant()); + ExecutionContext::local_default( + UserId::new("user").unwrap(), + ExtensionId::new("caller").unwrap(), + RuntimeKind::Script, + TrustClass::UserTrusted, + grants, + MountView::default(), + ) + .unwrap() +} + +fn execution_context_without_grants() -> ExecutionContext { + ExecutionContext::local_default( + UserId::new("user").unwrap(), + ExtensionId::new("caller").unwrap(), + RuntimeKind::Script, + TrustClass::UserTrusted, + CapabilitySet::default(), + MountView::default(), + ) + .unwrap() +} + +fn dispatch_grant() -> CapabilityGrant { + CapabilityGrant { + id: CapabilityGrantId::new(), + capability: script_capability_id(), + grantee: Principal::Extension(ExtensionId::new("caller").unwrap()), + issued_by: Principal::HostRuntime, + constraints: GrantConstraints { + allowed_effects: vec![EffectKind::DispatchCapability], + mounts: MountView::default(), + network: NetworkPolicy::default(), + secrets: Vec::new(), + resource_ceiling: None, + expires_at: None, + max_invocations: None, + }, + } +} + +fn local_manifest_trust_policy() -> HostTrustPolicy { + HostTrustPolicy::new(vec![Box::new(AdminConfig::with_entries(vec![ + AdminEntry::for_local_manifest( + PackageId::new("script").unwrap(), + "/system/extensions/script/manifest.toml".to_string(), + None, + HostTrustAssignment::user_trusted(), + vec![EffectKind::DispatchCapability], + None, + ), + ]))]) + .unwrap() +} + +fn trust_decision_with_dispatch_authority() -> TrustDecision { + TrustDecision { + effective_trust: EffectiveTrustClass::user_trusted(), + authority_ceiling: AuthorityCeiling { + allowed_effects: vec![EffectKind::DispatchCapability], + max_resource_ceiling: None, + }, + provenance: TrustProvenance::Default, + evaluated_at: Utc::now(), + } +} + +fn sample_scope(invocation_id: InvocationId) -> ResourceScope { + ResourceScope { + tenant_id: TenantId::new("tenant1").unwrap(), + user_id: UserId::new("user1").unwrap(), + agent_id: None, + project_id: None, + mission_id: None, + thread_id: None, + invocation_id, + } +} + +fn sample_policy() -> NetworkPolicy { + NetworkPolicy { + allowed_targets: vec![NetworkTargetPattern { + scheme: Some(NetworkScheme::Https), + host_pattern: "api.example.test".to_string(), + port: None, + }], + deny_private_ip_ranges: true, + max_egress_bytes: Some(4096), + } +} + +fn caller_supplied_policy() -> NetworkPolicy { + NetworkPolicy { + allowed_targets: vec![NetworkTargetPattern { + scheme: Some(NetworkScheme::Https), + host_pattern: "caller.example.test".to_string(), + port: None, + }], + deny_private_ip_ranges: false, + max_egress_bytes: Some(1), + } +} + +fn script_capability_id() -> CapabilityId { + CapabilityId::new("script.echo").unwrap() +} + +fn assert_event_kinds(events: &InMemoryEventSink, expected: &[RuntimeEventKind]) { + let actual = events + .events() + .into_iter() + .map(|event| event.kind) + .collect::>(); + assert_eq!(actual, expected); +} + +fn assert_failed_outcome(outcome: RuntimeCapabilityOutcome, expected: RuntimeFailureKind) { + match outcome { + RuntimeCapabilityOutcome::Failed(failure) => assert_eq!(failure.kind, expected), + other => panic!("expected failed outcome {expected:?}, got {other:?}"), + } +} + +const SCRIPT_MANIFEST: &str = r#" +id = "script" +name = "Script Echo" +version = "0.1.0" +description = "Script echo test extension" +trust = "third_party" + +[runtime] +kind = "script" +runner = "sandboxed_process" +command = "echo-script" +args = [] + +[[capabilities]] +id = "script.echo" +description = "Echo text through script runtime" +effects = ["dispatch_capability"] +default_permission = "allow" +parameters_schema = { type = "object" } +"#; diff --git a/docs/reborn/harness/e2e.md b/docs/reborn/harness/e2e.md new file mode 100644 index 0000000000..65b3db8062 --- /dev/null +++ b/docs/reborn/harness/e2e.md @@ -0,0 +1,146 @@ +# Reborn E2E Harness + +This document is the branch-local map for the dedicated Reborn E2E gate. Reborn is not missing tests: `main` already contains extensive crate-level contract and integration coverage. The gap this branch closes is a single, named E2E workflow that runs the Reborn architecture spine together and keeps a small product-surface smoke check beside it. + +## What already exists + +| Area | Existing coverage | +| --- | --- | +| Architecture boundaries | `crates/ironclaw_architecture/tests/reborn_dependency_boundaries.rs` | +| Host runtime facade and outcomes | `crates/ironclaw_host_runtime/tests/host_runtime_contract.rs` | +| Host runtime production composition | `crates/ironclaw_host_runtime/tests/host_runtime_services_contract.rs` | +| Dedicated Reborn E2E spine | `crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs` | +| Capability host invoke/resume/spawn | `crates/ironclaw_capabilities/tests/capability_host_*` | +| Dispatcher adapter selection | `crates/ironclaw_dispatcher/tests/vertical_slice_contract.rs` | +| WASM runtime lane | `crates/ironclaw_wasm/tests/wasm_dispatch_integration.rs` and `wasm_http_adapter_contract.rs` | +| Script runtime lane | `crates/ironclaw_scripts/tests/script_dispatch_integration.rs` and `script_http_adapter_contract.rs` | +| MCP runtime lane | `crates/ironclaw_mcp/tests/mcp_dispatch_integration.rs` and `mcp_adapter_contract.rs` | +| Process lifecycle | `crates/ironclaw_processes/tests/process_dispatch_integration.rs` and process service/store contracts | +| Network policy and host HTTP egress | `crates/ironclaw_network/tests/*` plus host-runtime HTTP egress tests | +| Secret storage/leases | `crates/ironclaw_secrets/tests/secret_store_contract.rs` plus host-runtime staged-secret tests | +| Events/audit replay | `crates/ironclaw_events/tests/durable_log_contract.rs` and host-runtime durable-event tests | +| Gateway product smoke | Existing Playwright scenarios under `tests/e2e/scenarios/`, especially `test_v2_*` | + +## Dedicated Reborn E2E goal + +The dedicated gate should answer one question: + +```text +Can the Reborn architecture path still execute happy, blocked, denied, failed, +background, network, secret, event, and product-smoke paths after a change? +``` + +It intentionally reuses existing deterministic contract/integration tests instead of duplicating them in a second test framework. + +## Happy path spine + +The Reborn E2E happy path is covered by a dedicated `reborn_e2e_gate.rs` spine test plus the broader host-runtime and runtime-lane tests: + +```text +Extension manifests +-> ExtensionRegistry +-> HostRuntimeServices +-> DefaultHostRuntime / HostRuntime facade +-> CapabilityHost authorization and run-state lifecycle +-> RuntimeDispatcher adapter selection +-> WASM / Script / MCP runtime adapters +-> resource reservation and reconciliation +-> durable runtime events +-> structured outcome returned to caller +``` + +Required assertions across the suite: + +- visible capability surfaces include expected descriptors and stable surface version; +- health reports missing runtime backends fail-closed and configured backends ready; +- authorized invocations reach the selected runtime adapter; +- resource reservations are reconciled or released; +- run-state reaches the expected terminal or blocked state; +- durable events are replayable and metadata-only; +- runtime output is structured JSON and redacted where obligations require it. + +## Other paths + +The dedicated gate includes new Reborn E2E gate tests plus existing tests for these non-happy paths: + +### Authorization and approval + +- denied authorization fails before dispatch; +- approval-required invocation blocks with a persisted approval request id; +- approved resume consumes the exact lease once; +- changed input, wrong scope/user, expired lease, or missing stores fail before dispatch; +- unsupported obligations fail closed. + +### Runtime availability and adapter failures + +- missing runtime backend reports a stable missing-runtime failure; +- runtime lane errors map to stable failure categories; +- dispatcher/runtime adapter boundaries remain dependency-clean. + +### Resource, process, and cancellation + +- reservations are released on failure and reconciled on success; +- spawned background processes publish started/completed/failed/killed transitions; +- cancellation reaches the process graph; +- late completion after kill does not publish a misleading success; +- process handoffs for the same scoped capability fail closed while active. + +### Network and secrets + +- runtime HTTP egress is host-mediated; +- missing staged network policy fails before transport; +- staged secret material is consumed once; +- runtime-supplied manual credentials are rejected; +- raw secrets, credential-shaped values, and private host paths are not exposed in runtime-visible output, errors, events, or audit records. + +### Event and observability + +- durable event cursors replay runtime events; +- stale/gap cursor behavior is deterministic; +- event/audit records carry correlation metadata without raw payload leaks. + +### Product smoke + +A small Playwright smoke scenario starts an isolated `ENGINE_V2=true` gateway with the mock LLM and verifies: + +- authenticated web shell loads; +- text-only chat completes and persists; +- tool-capable prompt completes through the gateway history path; +- no duplicate assistant response is emitted for a single user turn. + +This smoke test does not replace the full browser E2E workflow. It proves the branch remains product-bootable while the Rust Reborn gate proves architecture behavior. + +## Local commands + +Run the deterministic Rust Reborn gate: + +```bash +# Full gate +scripts/reborn-e2e-rust.sh + +# Or run one CI matrix group at a time +scripts/reborn-e2e-rust.sh architecture +scripts/reborn-e2e-rust.sh runtimes +scripts/reborn-e2e-rust.sh substrates +``` + +The script expands to the dedicated `reborn_e2e_gate.rs` tests plus the current Reborn boundary, host-runtime, capability-host, dispatcher, WASM, Script, MCP, process, event, filesystem, network, secret, resource, run-state, approval, and authorization contract tests. Use the script as the source of truth for local/CI parity rather than copying individual `cargo test` commands. + +Run the gateway smoke test: + +```bash +cargo build --no-default-features --features libsql +cd tests/e2e +pip install -e . +playwright install --with-deps chromium # on Linux CI; local macOS can omit --with-deps +pytest scenarios/test_reborn_gateway_smoke.py -v --timeout=120 +``` + +## CI ownership + +`reborn-e2e.yml` is intentionally separate from `e2e.yml`: + +- Reborn changes can run a focused architecture gate without destabilizing the main browser E2E matrix. +- The workflow is advisory by default and intentionally does not run on `merge_group`; add a merge-queue trigger only after the gate proves stable and is deliberately promoted to branch protection. +- The Rust jobs should stay deterministic and avoid live providers. +- The gateway job should remain a smoke test, not a second full browser matrix. diff --git a/scripts/reborn-e2e-rust.sh b/scripts/reborn-e2e-rust.sh new file mode 100755 index 0000000000..467178175f --- /dev/null +++ b/scripts/reborn-e2e-rust.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Run the deterministic Rust-side Reborn E2E gate. +# Usage: +# scripts/reborn-e2e-rust.sh # all groups +# scripts/reborn-e2e-rust.sh architecture # boundary + host runtime spine +# scripts/reborn-e2e-rust.sh runtimes # dispatcher/runtime/process lanes +# scripts/reborn-e2e-rust.sh substrates # event/network/secret substrates +# +# Extra cargo test args can be passed through CARGO_TEST_ARGS, for example: +# CARGO_TEST_ARGS='-- --nocapture' scripts/reborn-e2e-rust.sh architecture + +group="${1:-all}" +extra_args=${CARGO_TEST_ARGS:-"-- --nocapture"} + +run_test() { + local package="$1" + local test_name="$2" + echo "::group::cargo test -p ${package} --test ${test_name}" + # shellcheck disable=SC2086 # extra_args intentionally expands into cargo's trailing args. + cargo test -p "${package}" --test "${test_name}" ${extra_args} + echo "::endgroup::" +} + +run_architecture() { + run_test ironclaw_architecture reborn_dependency_boundaries + run_test ironclaw_host_runtime host_runtime_contract + run_test ironclaw_host_runtime host_runtime_services_contract + run_test ironclaw_host_runtime reborn_e2e_gate + run_test ironclaw_host_runtime reborn_invoke_vertical_slice + run_test ironclaw_host_runtime runtime_http_egress_contract + run_test ironclaw_host_runtime builtin_obligation_handler_contract + run_test ironclaw_host_runtime obligation_services_composition_contract + run_test ironclaw_host_runtime production_trust_contract + run_test ironclaw_capabilities capability_boundary_contract + run_test ironclaw_capabilities capability_host_contract + run_test ironclaw_capabilities capability_host_dispatcher_integration + run_test ironclaw_capabilities capability_host_process_integration + run_test ironclaw_capabilities capability_host_run_state_contract + run_test ironclaw_capabilities capability_host_spawn_contract + run_test ironclaw_capabilities capability_obligation_handler_contract +} + +run_runtimes() { + run_test ironclaw_dispatcher boundary_contract + run_test ironclaw_dispatcher dispatch_contract + run_test ironclaw_dispatcher event_dispatch_contract + run_test ironclaw_dispatcher runtime_dispatcher_integration + run_test ironclaw_dispatcher vertical_slice_contract + run_test ironclaw_wasm wasm_dispatch_integration + run_test ironclaw_wasm wasm_http_adapter_contract + run_test ironclaw_wasm wit_tool_runtime_contract + run_test ironclaw_scripts script_dispatch_integration + run_test ironclaw_scripts script_http_adapter_contract + run_test ironclaw_scripts script_runner_contract + run_test ironclaw_mcp mcp_adapter_contract + run_test ironclaw_mcp mcp_dispatch_integration + run_test ironclaw_processes process_dispatch_integration + run_test ironclaw_processes process_host_contract + run_test ironclaw_processes process_services_contract + run_test ironclaw_processes process_store_contract +} + +run_substrates() { + run_test ironclaw_events durable_log_contract + run_test ironclaw_filesystem catalog_contract + run_test ironclaw_filesystem filesystem_contract + run_test ironclaw_network boundary_contract + run_test ironclaw_network network_http_egress_contract + run_test ironclaw_network network_policy_contract + run_test ironclaw_secrets boundary_contract + run_test ironclaw_secrets secret_store_contract + run_test ironclaw_resources resource_governor_contract + run_test ironclaw_run_state approval_resolution_contract + run_test ironclaw_run_state run_state_contract + run_test ironclaw_approvals approval_resolution_contract + run_test ironclaw_approvals boundary_contract + run_test ironclaw_authorization boundary_contract + run_test ironclaw_authorization capability_access_contract + run_test ironclaw_authorization capability_lease_contract +} + +case "${group}" in + architecture) + run_architecture + ;; + runtimes) + run_runtimes + ;; + substrates) + run_substrates + ;; + all) + run_architecture + run_runtimes + run_substrates + ;; + *) + echo "unknown Reborn E2E group: ${group}" >&2 + echo "expected one of: architecture, runtimes, substrates, all" >&2 + exit 2 + ;; +esac diff --git a/tests/e2e/scenarios/test_reborn_gateway_smoke.py b/tests/e2e/scenarios/test_reborn_gateway_smoke.py new file mode 100644 index 0000000000..7533b3a308 --- /dev/null +++ b/tests/e2e/scenarios/test_reborn_gateway_smoke.py @@ -0,0 +1,265 @@ +"""Dedicated Reborn gateway smoke E2E. + +This is intentionally small. The Rust Reborn gate proves the host/runtime +architecture. This Playwright/API smoke test proves the reborn-main branch still +boots an isolated ENGINE_V2 gateway, serves the browser shell, persists a normal +chat turn, and completes a simple tool-capable turn without duplicate terminal +assistant responses. +""" + +import asyncio +import os +import signal +import socket +import tempfile + +import pytest +from playwright.async_api import expect + +from helpers import AUTH_TOKEN, SEL, api_get, api_post, wait_for_ready + +_REBORN_GATEWAY_DB_TMPDIR = tempfile.TemporaryDirectory(prefix="ironclaw-reborn-gateway-e2e-") +_REBORN_GATEWAY_HOME_TMPDIR = tempfile.TemporaryDirectory(prefix="ironclaw-reborn-gateway-e2e-home-") + + +def _reserve_loopback_sockets(count: int) -> list[socket.socket]: + sockets: list[socket.socket] = [] + try: + while len(sockets) < count: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.bind(("127.0.0.1", 0)) + sockets.append(sock) + return sockets + except Exception: + for sock in sockets: + sock.close() + raise + + +def _close_reserved_sockets(sockets: list[socket.socket]) -> None: + for sock in sockets: + if sock.fileno() != -1: + sock.close() + + +def _forward_coverage_env(env: dict[str, str]) -> None: + for key, value in os.environ.items(): + if key.startswith(("CARGO_LLVM_COV", "LLVM_")) or key in { + "CARGO_ENCODED_RUSTFLAGS", + "CARGO_INCREMENTAL", + }: + env[key] = value + + +async def _stop_process(proc, *, sig=signal.SIGINT, timeout: float = 10) -> None: + async def _drain_pipes() -> None: + try: + await asyncio.wait_for(proc.communicate(), timeout=1) + except (asyncio.TimeoutError, ValueError): + pass + + if proc.returncode is not None: + await _drain_pipes() + return + + try: + proc.send_signal(sig) + except ProcessLookupError: + await _drain_pipes() + return + + try: + await asyncio.wait_for(proc.wait(), timeout=timeout) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + await _drain_pipes() + + +@pytest.fixture(scope="module") +async def reborn_gateway_server(ironclaw_binary, mock_llm_server): + """Start an isolated gateway configured for the Reborn/V2 product shell.""" + home_dir = _REBORN_GATEWAY_HOME_TMPDIR.name + base_dir = os.path.join(home_dir, ".ironclaw") + os.makedirs(base_dir, exist_ok=True) + reserved_sockets = _reserve_loopback_sockets(2) + gateway_port = reserved_sockets[0].getsockname()[1] + http_port = reserved_sockets[1].getsockname()[1] + + env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": home_dir, + "IRONCLAW_BASE_DIR": base_dir, + "RUST_LOG": "ironclaw=info", + "RUST_BACKTRACE": "1", + "ENGINE_V2": "true", + "AGENT_AUTO_APPROVE_TOOLS": "true", + "GATEWAY_ENABLED": "true", + "GATEWAY_HOST": "127.0.0.1", + "GATEWAY_PORT": str(gateway_port), + "GATEWAY_AUTH_TOKEN": AUTH_TOKEN, + "GATEWAY_USER_ID": "reborn-gateway-e2e-user", + "HTTP_HOST": "127.0.0.1", + "HTTP_PORT": str(http_port), + "CLI_ENABLED": "false", + "LLM_BACKEND": "openai_compatible", + "LLM_BASE_URL": mock_llm_server, + "LLM_API_KEY": "mock-api-key", + "LLM_MODEL": "mock-model", + "DATABASE_BACKEND": "libsql", + "LIBSQL_PATH": os.path.join(_REBORN_GATEWAY_DB_TMPDIR.name, "reborn-gateway-e2e.db"), + "SANDBOX_ENABLED": "false", + "SKILLS_ENABLED": "false", + "ROUTINES_ENABLED": "false", + "HEARTBEAT_ENABLED": "false", + "EMBEDDING_ENABLED": "false", + "WASM_ENABLED": "false", + "ONBOARD_COMPLETED": "true", + } + _forward_coverage_env(env) + + _close_reserved_sockets(reserved_sockets) + proc = await asyncio.create_subprocess_exec( + ironclaw_binary, + "--no-onboard", + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=env, + ) + base_url = f"http://127.0.0.1:{gateway_port}" + + try: + await wait_for_ready(f"{base_url}/api/health", timeout=60) + yield base_url + except TimeoutError: + if proc.returncode is None: + await _stop_process(proc, timeout=2) + stderr_text = "" + if proc.stderr: + try: + stderr_bytes = await asyncio.wait_for(proc.stderr.read(8192), timeout=2) + stderr_text = stderr_bytes.decode("utf-8", errors="replace") + except asyncio.TimeoutError: + pass + pytest.fail( + f"Reborn gateway smoke server failed to start on port {gateway_port}.\n" + f"stderr:\n{stderr_text}" + ) + finally: + if proc.returncode is None: + await _stop_process(proc, sig=signal.SIGINT, timeout=10) + if proc.returncode is None: + await _stop_process(proc, sig=signal.SIGTERM, timeout=5) + + +@pytest.fixture +async def reborn_gateway_page(reborn_gateway_server, browser): + context = await browser.new_context(viewport={"width": 1280, "height": 720}) + page = await context.new_page() + await page.goto(f"{reborn_gateway_server}/?token={AUTH_TOKEN}") + await page.wait_for_selector(SEL["auth_screen"], state="hidden", timeout=15000) + await page.wait_for_function( + "() => typeof sseHasConnectedBefore !== 'undefined' && sseHasConnectedBefore === true", + timeout=10000, + ) + yield page + await context.close() + + +async def _create_thread(base_url: str) -> str: + response = await api_post(base_url, "/api/chat/thread/new", timeout=15) + response.raise_for_status() + return response.json()["id"] + + +async def _send_message(base_url: str, thread_id: str, content: str) -> None: + response = await api_post( + base_url, + "/api/chat/send", + json={"content": content, "thread_id": thread_id}, + timeout=30, + ) + assert response.status_code in (200, 202), response.text + + +async def _wait_for_terminal_turn( + base_url: str, + thread_id: str, + expected_user_input: str, + *, + timeout: float = 45.0, +) -> dict: + last_history = {} + for _ in range(int(timeout * 2)): + response = await api_get( + base_url, + f"/api/chat/history?thread_id={thread_id}", + timeout=15, + ) + response.raise_for_status() + history = response.json() + last_history = history + turns = history.get("turns", []) + matching_turns = [ + turn + for turn in turns + if expected_user_input in (turn.get("user_input") or "") + ] + if matching_turns and (matching_turns[-1].get("response") or "").strip(): + return matching_turns[-1] + await asyncio.sleep(0.5) + + raise AssertionError( + f"Timed out waiting for terminal turn containing {expected_user_input!r}. " + f"Last history: {last_history}" + ) + + +async def test_reborn_gateway_loads_engine_v2_shell(reborn_gateway_page): + """The isolated Reborn smoke gateway should boot the ENGINE_V2 shell.""" + chat_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="chat")) + missions_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="missions")) + routines_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="routines")) + + await expect(chat_tab).to_be_visible() + await expect(missions_tab).to_be_visible() + await expect(routines_tab).to_be_hidden() + + +async def test_reborn_gateway_persists_text_and_tool_turns_without_duplicate_response( + reborn_gateway_server, +): + """A text turn and an auto-approved tool turn should each produce one terminal response.""" + thread_id = await _create_thread(reborn_gateway_server) + + text_prompt = "reborn gateway smoke: what is 2+2?" + await _send_message(reborn_gateway_server, thread_id, text_prompt) + text_turn = await _wait_for_terminal_turn(reborn_gateway_server, thread_id, text_prompt) + assert "4" in text_turn.get("response", "") + + tool_prompt = "echo reborn gateway smoke tool result" + await _send_message(reborn_gateway_server, thread_id, tool_prompt) + tool_turn = await _wait_for_terminal_turn(reborn_gateway_server, thread_id, tool_prompt) + assert "reborn gateway smoke tool result" in tool_turn.get("response", "").lower() + + tool_calls = tool_turn.get("tool_calls", []) + assert tool_calls, f"Expected persisted tool call metadata, got: {tool_turn}" + assert any(call.get("name") == "echo" and call.get("has_result") for call in tool_calls) + + history_response = await api_get( + reborn_gateway_server, + f"/api/chat/history?thread_id={thread_id}", + timeout=15, + ) + history_response.raise_for_status() + matching_tool_turns = [ + turn + for turn in history_response.json().get("turns", []) + if tool_prompt in (turn.get("user_input") or "") + and (turn.get("response") or "").strip() + ] + assert len(matching_tool_turns) == 1, ( + "Expected one terminal assistant response for the tool prompt, got " + f"{len(matching_tool_turns)} turns: {matching_tool_turns}" + )