test(reborn): add dedicated e2e gate (#3251)

This commit is contained in:
firat.sertgoz
2026-05-07 10:55:55 +03:00
committed by GitHub
parent 5f4a4c7793
commit 7db0df6b0a
5 changed files with 1390 additions and 0 deletions

140
.github/workflows/reborn-e2e.yml vendored Normal file
View File

@@ -0,0 +1,140 @@
name: Reborn E2E
on:
workflow_call:
inputs:
ref:
description: Commit SHA or ref to test
required: false
type: string
workflow_dispatch:
pull_request:
branches:
- main
paths:
- "crates/ironclaw_*/**"
- "docs/reborn/**"
- "scripts/reborn-e2e-rust.sh"
- "tests/e2e/**"
- "build.rs"
- "providers.json"
- "Cargo.toml"
- "Cargo.lock"
- ".github/workflows/reborn-e2e.yml"
push:
branches:
- main
paths:
- "crates/ironclaw_*/**"
- "docs/reborn/**"
- "scripts/reborn-e2e-rust.sh"
- "tests/e2e/**"
- "build.rs"
- "providers.json"
- "Cargo.toml"
- "Cargo.lock"
- ".github/workflows/reborn-e2e.yml"
permissions:
contents: read
concurrency:
group: reborn-e2e-${{ github.event_name }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
jobs:
rust-reborn:
name: Rust Reborn (${{ matrix.group }})
runs-on: ubuntu-latest
timeout-minutes: 35
strategy:
fail-fast: false
matrix:
group:
- architecture
- runtimes
- substrates
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
ref: ${{ inputs.ref || github.sha }}
persist-credentials: false
- name: Install Rust
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- name: Restore Rust cache
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
key: reborn-e2e-${{ matrix.group }}
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
- name: Run deterministic Reborn Rust gate
env:
CARGO_TEST_ARGS: "-- --nocapture"
run: scripts/reborn-e2e-rust.sh ${{ matrix.group }}
gateway-smoke:
name: Reborn gateway smoke
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
ref: ${{ inputs.ref || github.sha }}
persist-credentials: false
- name: Install Rust
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
- name: Restore Rust cache
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
with:
key: reborn-e2e-gateway-smoke
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
- name: Build ironclaw with libSQL
run: cargo build --no-default-features --features libsql
- name: Install Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: "3.12"
- name: Install E2E dependencies
run: |
cd tests/e2e
pip install -e .
playwright install --with-deps chromium
- name: Run Reborn gateway smoke
run: pytest tests/e2e/scenarios/test_reborn_gateway_smoke.py -v --timeout=120
- name: Upload screenshots on failure
if: failure()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: reborn-e2e-screenshots
path: tests/e2e/screenshots/
if-no-files-found: ignore
reborn-e2e:
name: Reborn E2E
runs-on: ubuntu-latest
if: always()
needs:
- rust-reborn
- gateway-smoke
steps:
- name: Check Reborn E2E jobs
run: |
if [[ "${{ needs.rust-reborn.result }}" != "success" ]]; then
echo "Rust Reborn E2E failed"
exit 1
fi
if [[ "${{ needs.gateway-smoke.result }}" != "success" ]]; then
echo "Reborn gateway smoke failed"
exit 1
fi

View File

@@ -0,0 +1,735 @@
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use chrono::Utc;
use ironclaw_approvals::LeaseApproval;
use ironclaw_authorization::{
CapabilityLeaseStatus, CapabilityLeaseStore, GrantAuthorizer, InMemoryCapabilityLeaseStore,
TrustAwareCapabilityDispatchAuthorizer,
};
use ironclaw_events::{
DurableEventLog, EventStreamKey, InMemoryDurableEventLog, InMemoryEventSink, ReadScope,
RuntimeEventKind,
};
use ironclaw_extensions::{ExtensionManifest, ExtensionPackage, ExtensionRegistry};
use ironclaw_filesystem::LocalFilesystem;
use ironclaw_host_api::*;
use ironclaw_host_runtime::{
CapabilitySurfaceVersion, HostHttpEgressService, HostRuntime, HostRuntimeServices,
NetworkObligationPolicyStore, RuntimeCapabilityOutcome, RuntimeCapabilityRequest,
RuntimeCapabilityResumeRequest, RuntimeFailureKind, RuntimeSecretInjectionStore,
RuntimeStatusRequest, SurfaceKind,
};
use ironclaw_network::{
NetworkHttpEgress, NetworkHttpError, NetworkHttpRequest, NetworkHttpResponse, NetworkUsage,
};
use ironclaw_processes::{InMemoryProcessResultStore, InMemoryProcessStore, ProcessServices};
use ironclaw_resources::{InMemoryResourceGovernor, ResourceAccount, ResourceTally};
use ironclaw_run_state::{
InMemoryApprovalRequestStore, InMemoryRunStateStore, RunStateStore, RunStatus,
};
use ironclaw_scripts::{
ScriptBackend, ScriptBackendOutput, ScriptBackendRequest, ScriptRuntime, ScriptRuntimeConfig,
};
use ironclaw_secrets::{InMemorySecretStore, SecretMaterial};
use ironclaw_trust::{
AdminConfig, AdminEntry, AuthorityCeiling, EffectiveTrustClass, HostTrustAssignment,
HostTrustPolicy, TrustDecision, TrustProvenance,
};
use serde_json::json;
#[tokio::test]
async fn reborn_e2e_gate_invokes_script_through_host_runtime_with_status_events_and_resources() {
let governor = Arc::new(InMemoryResourceGovernor::new());
let run_state = Arc::new(InMemoryRunStateStore::new());
let event_log = Arc::new(InMemoryDurableEventLog::new());
let services = HostRuntimeServices::new(
Arc::new(registry_with_manifest(SCRIPT_MANIFEST)),
Arc::new(LocalFilesystem::new()),
Arc::clone(&governor),
Arc::new(GrantAuthorizer::new()),
ProcessServices::in_memory(),
CapabilitySurfaceVersion::new("surface-v1").unwrap(),
)
.with_trust_policy(Arc::new(local_manifest_trust_policy()))
.with_run_state(Arc::clone(&run_state))
.with_script_runtime(Arc::new(ScriptRuntime::new(
ScriptRuntimeConfig::for_testing(),
EchoScriptBackend,
)))
.with_durable_event_log(Arc::clone(&event_log));
let runtime = services.host_runtime();
let context = execution_context_with_dispatch_grant();
let scope = context.resource_scope.clone();
let invocation_id = context.invocation_id;
let surface = runtime
.visible_capabilities(ironclaw_host_runtime::VisibleCapabilityRequest::new(
scope.clone(),
context.correlation_id,
SurfaceKind::new("gateway-smoke").unwrap(),
))
.await
.unwrap();
assert_eq!(surface.version.as_str(), "surface-v1");
assert_eq!(surface.descriptors.len(), 1);
assert_eq!(surface.descriptors[0].id, script_capability_id());
let health = runtime.health().await.unwrap();
assert!(health.ready);
assert!(health.missing_runtime_backends.is_empty());
let status_before = runtime
.runtime_status(RuntimeStatusRequest::new(
scope.clone(),
CorrelationId::new(),
))
.await
.unwrap();
assert!(status_before.active_work.is_empty());
let input = json!({
"message": "reborn e2e happy path",
"secret_sentinel": "SECRET_REBORN_E2E_GATE_SHOULD_NOT_LEAK",
"host_path_sentinel": "/private/tmp/reborn-e2e-gate"
});
let outcome = runtime
.invoke_capability(RuntimeCapabilityRequest::new(
context,
script_capability_id(),
ResourceEstimate {
output_bytes: Some(4096),
..ResourceEstimate::default()
},
input.clone(),
trust_decision_with_dispatch_authority(),
))
.await
.unwrap();
match outcome {
RuntimeCapabilityOutcome::Completed(completed) => {
assert_eq!(completed.capability_id, script_capability_id());
assert_eq!(completed.output, input);
assert!(completed.usage.output_bytes > 0);
}
other => panic!("expected completed outcome, got {other:?}"),
}
let run = run_state.get(&scope, invocation_id).await.unwrap().unwrap();
assert_eq!(run.status, RunStatus::Completed);
let tenant_account = ResourceAccount::tenant(scope.tenant_id.clone());
assert_eq!(
governor.reserved_for(&tenant_account),
ResourceTally::default()
);
assert!(governor.usage_for(&tenant_account).output_bytes > 0);
let status_after = runtime
.runtime_status(RuntimeStatusRequest::new(
scope.clone(),
CorrelationId::new(),
))
.await
.unwrap();
assert!(status_after.active_work.is_empty());
let replay = event_log
.read_after_cursor(
&EventStreamKey::from_scope(&scope),
&ReadScope::any(),
None,
10,
)
.await
.unwrap();
let kinds = replay
.entries
.iter()
.map(|entry| entry.record.kind)
.collect::<Vec<_>>();
assert_eq!(
kinds,
vec![
RuntimeEventKind::DispatchRequested,
RuntimeEventKind::RuntimeSelected,
RuntimeEventKind::DispatchSucceeded,
]
);
let serialized = serde_json::to_string(&replay).unwrap();
for forbidden in [
"SECRET_REBORN_E2E_GATE_SHOULD_NOT_LEAK",
"/private/tmp/reborn-e2e-gate",
] {
assert!(
!serialized.contains(forbidden),
"durable Reborn E2E event replay leaked {forbidden}: {serialized}"
);
}
}
#[tokio::test]
async fn reborn_e2e_gate_blocks_for_approval_resumes_once_and_rejects_replay() {
let fixture = approval_resume_fixture();
let runtime = fixture.services.host_runtime();
let context = execution_context_without_grants();
let scope = context.resource_scope.clone();
let invocation_id = context.invocation_id;
let input = json!({"message": "approval resume through Reborn E2E gate"});
let gate = block_for_approval(&runtime, context.clone(), input.clone()).await;
let blocked_run = fixture
.run_state
.get(&scope, invocation_id)
.await
.unwrap()
.unwrap();
assert_eq!(blocked_run.status, RunStatus::BlockedApproval);
assert_eq!(
blocked_run.approval_request_id,
Some(gate.approval_request_id)
);
let lease =
approve_dispatch_for_services(&fixture.services, &scope, gate.approval_request_id).await;
let resumed = runtime
.resume_capability(RuntimeCapabilityResumeRequest::new(
context.clone(),
gate.approval_request_id,
script_capability_id(),
ResourceEstimate::default(),
input.clone(),
trust_decision_with_dispatch_authority(),
))
.await
.unwrap();
match resumed {
RuntimeCapabilityOutcome::Completed(completed) => {
assert_eq!(completed.capability_id, script_capability_id());
assert_eq!(completed.output, input);
}
other => panic!("expected completed approval resume, got {other:?}"),
}
assert_eq!(
fixture
.capability_leases
.get(&scope, lease.grant.id)
.await
.unwrap()
.status,
CapabilityLeaseStatus::Consumed
);
assert_event_kinds(
&fixture.events,
&[
RuntimeEventKind::DispatchRequested,
RuntimeEventKind::RuntimeSelected,
RuntimeEventKind::DispatchSucceeded,
],
);
let replay = runtime
.resume_capability(RuntimeCapabilityResumeRequest::new(
context,
gate.approval_request_id,
script_capability_id(),
ResourceEstimate::default(),
json!({"message": "approval resume through Reborn E2E gate"}),
trust_decision_with_dispatch_authority(),
))
.await
.unwrap();
assert_failed_outcome(replay, RuntimeFailureKind::Authorization);
assert_eq!(
fixture.events.events().len(),
3,
"replayed approval resume must fail before a second runtime dispatch"
);
}
#[tokio::test]
async fn reborn_e2e_gate_fails_unsupported_obligations_before_runtime_events_or_success() {
let run_state = Arc::new(InMemoryRunStateStore::new());
let events = InMemoryEventSink::new();
let governor = Arc::new(InMemoryResourceGovernor::new());
let services = HostRuntimeServices::new(
Arc::new(registry_with_manifest(SCRIPT_MANIFEST)),
Arc::new(LocalFilesystem::new()),
Arc::clone(&governor),
Arc::new(ObligatingAuthorizer),
ProcessServices::in_memory(),
CapabilitySurfaceVersion::new("surface-v1").unwrap(),
)
.with_trust_policy(Arc::new(local_manifest_trust_policy()))
.with_run_state(Arc::clone(&run_state))
.with_script_runtime(Arc::new(ScriptRuntime::new(
ScriptRuntimeConfig::for_testing(),
EchoScriptBackend,
)))
.with_event_sink(Arc::new(events.clone()));
let runtime = services.host_runtime();
let context = execution_context_with_dispatch_grant();
let scope = context.resource_scope.clone();
let invocation_id = context.invocation_id;
let outcome = runtime
.invoke_capability(RuntimeCapabilityRequest::new(
context,
script_capability_id(),
ResourceEstimate::default(),
json!({"message": "unsupported obligation"}),
trust_decision_with_dispatch_authority(),
))
.await
.unwrap();
assert_failed_outcome(outcome, RuntimeFailureKind::Backend);
assert!(events.events().is_empty());
let run = run_state.get(&scope, invocation_id).await.unwrap().unwrap();
assert_eq!(run.status, RunStatus::Failed);
assert_eq!(run.error_kind.as_deref(), Some("ObligationFailed"));
let tenant_account = ResourceAccount::tenant(scope.tenant_id.clone());
assert_eq!(
governor.reserved_for(&tenant_account),
ResourceTally::default()
);
assert_eq!(
governor.usage_for(&tenant_account),
ResourceTally::default()
);
}
#[test]
fn reborn_e2e_gate_host_http_consumes_staged_policy_and_secret_once() {
let network = RecordingNetwork::ok(NetworkHttpResponse {
status: 200,
headers: vec![],
body: br#"{"ok":true}"#.to_vec(),
usage: NetworkUsage {
request_bytes: 5,
response_bytes: 11,
resolved_ip: None,
},
});
let network_recorder = network.requests.clone();
let policy_store = Arc::new(NetworkObligationPolicyStore::new());
let secret_injections = Arc::new(RuntimeSecretInjectionStore::new());
let scope = sample_scope(InvocationId::new());
let capability_id = script_capability_id();
let handle = SecretHandle::new("api-token").unwrap();
let staged_policy = sample_policy();
policy_store.insert(&scope, &capability_id, staged_policy.clone());
secret_injections
.insert(
&scope,
&capability_id,
&handle,
SecretMaterial::from("sk-reborn-e2e-staged-secret"),
)
.unwrap();
let service = HostHttpEgressService::new(network, InMemorySecretStore::new())
.with_network_policy_store(Arc::clone(&policy_store))
.with_secret_injection_store(Arc::clone(&secret_injections));
let request = RuntimeHttpEgressRequest {
runtime: RuntimeKind::Script,
scope: scope.clone(),
capability_id: capability_id.clone(),
method: NetworkMethod::Post,
url: "https://api.example.test/v1/run".to_string(),
headers: vec![],
body: b"hello".to_vec(),
network_policy: caller_supplied_policy(),
credential_injections: vec![RuntimeCredentialInjection {
handle: handle.clone(),
source: RuntimeCredentialSource::StagedObligation {
capability_id: capability_id.clone(),
},
target: RuntimeCredentialTarget::Header {
name: "authorization".to_string(),
prefix: Some("Bearer ".to_string()),
},
required: true,
}],
response_body_limit: Some(4096),
timeout_ms: None,
};
let response = service
.execute(request.clone())
.expect("host HTTP egress should use staged Reborn policy and secret material");
assert_eq!(response.status, 200);
let recorded = network_recorder.lock().unwrap();
assert_eq!(recorded.len(), 1);
assert_eq!(recorded[0].policy, staged_policy);
assert_eq!(
recorded[0]
.headers
.iter()
.find(|(name, _)| name == "authorization"),
Some(&(
"authorization".to_string(),
"Bearer sk-reborn-e2e-staged-secret".to_string()
))
);
drop(recorded);
assert!(
secret_injections
.take(&scope, &capability_id, &handle)
.unwrap()
.is_none(),
"staged secret material must be consumed exactly once"
);
assert_eq!(
policy_store.get(&scope, &capability_id),
Some(staged_policy),
"host egress must leave staged network policy for invocation/process lifecycle cleanup"
);
let replay = service
.execute(request)
.expect_err("consumed staged secret must not be reusable");
assert!(matches!(replay, RuntimeHttpEgressError::Credential { .. }));
assert_eq!(
network_recorder.lock().unwrap().len(),
1,
"replay must fail before a second outbound transport attempt"
);
}
type InMemoryServices = HostRuntimeServices<
LocalFilesystem,
InMemoryResourceGovernor,
InMemoryProcessStore,
InMemoryProcessResultStore,
>;
struct ApprovalFixture {
services: InMemoryServices,
run_state: Arc<InMemoryRunStateStore>,
capability_leases: Arc<InMemoryCapabilityLeaseStore>,
events: InMemoryEventSink,
}
fn approval_resume_fixture() -> ApprovalFixture {
let run_state = Arc::new(InMemoryRunStateStore::new());
let approval_requests = Arc::new(InMemoryApprovalRequestStore::new());
let capability_leases = Arc::new(InMemoryCapabilityLeaseStore::new());
let events = InMemoryEventSink::new();
let services = HostRuntimeServices::new(
Arc::new(registry_with_manifest(SCRIPT_MANIFEST)),
Arc::new(LocalFilesystem::new()),
Arc::new(InMemoryResourceGovernor::new()),
Arc::new(ApprovalThenGrantAuthorizer),
ProcessServices::in_memory(),
CapabilitySurfaceVersion::new("surface-v1").unwrap(),
)
.with_trust_policy(Arc::new(local_manifest_trust_policy()))
.with_run_state(Arc::clone(&run_state))
.with_approval_requests(approval_requests)
.with_capability_leases(Arc::clone(&capability_leases))
.with_script_runtime(Arc::new(ScriptRuntime::new(
ScriptRuntimeConfig::for_testing(),
EchoScriptBackend,
)))
.with_event_sink(Arc::new(events.clone()));
ApprovalFixture {
services,
run_state,
capability_leases,
events,
}
}
async fn block_for_approval(
runtime: &impl HostRuntime,
context: ExecutionContext,
input: serde_json::Value,
) -> ironclaw_host_runtime::RuntimeApprovalGate {
let outcome = runtime
.invoke_capability(RuntimeCapabilityRequest::new(
context,
script_capability_id(),
ResourceEstimate::default(),
input,
trust_decision_with_dispatch_authority(),
))
.await
.unwrap();
match outcome {
RuntimeCapabilityOutcome::ApprovalRequired(gate) => gate,
other => panic!("expected approval gate, got {other:?}"),
}
}
async fn approve_dispatch_for_services(
services: &InMemoryServices,
scope: &ResourceScope,
approval_request_id: ApprovalRequestId,
) -> ironclaw_authorization::CapabilityLease {
services
.approval_resolver()
.expect("approval resolver should be configured")
.approve_dispatch(
scope,
approval_request_id,
LeaseApproval {
issued_by: Principal::HostRuntime,
allowed_effects: vec![EffectKind::DispatchCapability],
mounts: MountView::default(),
network: NetworkPolicy::default(),
secrets: Vec::new(),
resource_ceiling: None,
expires_at: None,
max_invocations: Some(1),
},
)
.await
.unwrap()
}
struct ApprovalThenGrantAuthorizer;
#[async_trait]
impl TrustAwareCapabilityDispatchAuthorizer for ApprovalThenGrantAuthorizer {
async fn authorize_dispatch_with_trust(
&self,
context: &ExecutionContext,
descriptor: &CapabilityDescriptor,
estimate: &ResourceEstimate,
trust_decision: &TrustDecision,
) -> Decision {
if context.grants.grants.is_empty() {
Decision::RequireApproval {
request: ApprovalRequest {
id: ApprovalRequestId::new(),
correlation_id: context.correlation_id,
requested_by: Principal::Extension(context.extension_id.clone()),
action: Box::new(Action::Dispatch {
capability: descriptor.id.clone(),
estimated_resources: estimate.clone(),
}),
invocation_fingerprint: None,
reason: "approval required".to_string(),
reusable_scope: None,
},
}
} else {
GrantAuthorizer::new()
.authorize_dispatch_with_trust(context, descriptor, estimate, trust_decision)
.await
}
}
}
struct ObligatingAuthorizer;
#[async_trait]
impl TrustAwareCapabilityDispatchAuthorizer for ObligatingAuthorizer {
async fn authorize_dispatch_with_trust(
&self,
_context: &ExecutionContext,
_descriptor: &CapabilityDescriptor,
_estimate: &ResourceEstimate,
_trust_decision: &TrustDecision,
) -> Decision {
Decision::Allow {
obligations: Obligations::new(vec![Obligation::AuditBefore]).unwrap(),
}
}
}
struct EchoScriptBackend;
impl ScriptBackend for EchoScriptBackend {
fn execute(&self, request: ScriptBackendRequest) -> Result<ScriptBackendOutput, String> {
let value = serde_json::from_str(&request.stdin_json).map_err(|error| error.to_string())?;
Ok(ScriptBackendOutput::json(value))
}
}
#[derive(Clone)]
struct RecordingNetwork {
response: Result<NetworkHttpResponse, NetworkHttpError>,
requests: Arc<Mutex<Vec<NetworkHttpRequest>>>,
}
impl RecordingNetwork {
fn ok(response: NetworkHttpResponse) -> Self {
Self {
response: Ok(response),
requests: Arc::new(Mutex::new(Vec::new())),
}
}
}
impl NetworkHttpEgress for RecordingNetwork {
fn execute(
&self,
request: NetworkHttpRequest,
) -> Result<NetworkHttpResponse, NetworkHttpError> {
self.requests.lock().unwrap().push(request);
self.response.clone()
}
}
fn registry_with_manifest(manifest: &str) -> ExtensionRegistry {
let mut registry = ExtensionRegistry::new();
let manifest = ExtensionManifest::parse(manifest).unwrap();
let package = ExtensionPackage::from_manifest(
manifest,
VirtualPath::new("/system/extensions/script").unwrap(),
)
.unwrap();
registry.insert(package).unwrap();
registry
}
fn execution_context_with_dispatch_grant() -> ExecutionContext {
let mut grants = CapabilitySet::default();
grants.grants.push(dispatch_grant());
ExecutionContext::local_default(
UserId::new("user").unwrap(),
ExtensionId::new("caller").unwrap(),
RuntimeKind::Script,
TrustClass::UserTrusted,
grants,
MountView::default(),
)
.unwrap()
}
fn execution_context_without_grants() -> ExecutionContext {
ExecutionContext::local_default(
UserId::new("user").unwrap(),
ExtensionId::new("caller").unwrap(),
RuntimeKind::Script,
TrustClass::UserTrusted,
CapabilitySet::default(),
MountView::default(),
)
.unwrap()
}
fn dispatch_grant() -> CapabilityGrant {
CapabilityGrant {
id: CapabilityGrantId::new(),
capability: script_capability_id(),
grantee: Principal::Extension(ExtensionId::new("caller").unwrap()),
issued_by: Principal::HostRuntime,
constraints: GrantConstraints {
allowed_effects: vec![EffectKind::DispatchCapability],
mounts: MountView::default(),
network: NetworkPolicy::default(),
secrets: Vec::new(),
resource_ceiling: None,
expires_at: None,
max_invocations: None,
},
}
}
fn local_manifest_trust_policy() -> HostTrustPolicy {
HostTrustPolicy::new(vec![Box::new(AdminConfig::with_entries(vec![
AdminEntry::for_local_manifest(
PackageId::new("script").unwrap(),
"/system/extensions/script/manifest.toml".to_string(),
None,
HostTrustAssignment::user_trusted(),
vec![EffectKind::DispatchCapability],
None,
),
]))])
.unwrap()
}
fn trust_decision_with_dispatch_authority() -> TrustDecision {
TrustDecision {
effective_trust: EffectiveTrustClass::user_trusted(),
authority_ceiling: AuthorityCeiling {
allowed_effects: vec![EffectKind::DispatchCapability],
max_resource_ceiling: None,
},
provenance: TrustProvenance::Default,
evaluated_at: Utc::now(),
}
}
fn sample_scope(invocation_id: InvocationId) -> ResourceScope {
ResourceScope {
tenant_id: TenantId::new("tenant1").unwrap(),
user_id: UserId::new("user1").unwrap(),
agent_id: None,
project_id: None,
mission_id: None,
thread_id: None,
invocation_id,
}
}
fn sample_policy() -> NetworkPolicy {
NetworkPolicy {
allowed_targets: vec![NetworkTargetPattern {
scheme: Some(NetworkScheme::Https),
host_pattern: "api.example.test".to_string(),
port: None,
}],
deny_private_ip_ranges: true,
max_egress_bytes: Some(4096),
}
}
fn caller_supplied_policy() -> NetworkPolicy {
NetworkPolicy {
allowed_targets: vec![NetworkTargetPattern {
scheme: Some(NetworkScheme::Https),
host_pattern: "caller.example.test".to_string(),
port: None,
}],
deny_private_ip_ranges: false,
max_egress_bytes: Some(1),
}
}
fn script_capability_id() -> CapabilityId {
CapabilityId::new("script.echo").unwrap()
}
fn assert_event_kinds(events: &InMemoryEventSink, expected: &[RuntimeEventKind]) {
let actual = events
.events()
.into_iter()
.map(|event| event.kind)
.collect::<Vec<_>>();
assert_eq!(actual, expected);
}
fn assert_failed_outcome(outcome: RuntimeCapabilityOutcome, expected: RuntimeFailureKind) {
match outcome {
RuntimeCapabilityOutcome::Failed(failure) => assert_eq!(failure.kind, expected),
other => panic!("expected failed outcome {expected:?}, got {other:?}"),
}
}
const SCRIPT_MANIFEST: &str = r#"
id = "script"
name = "Script Echo"
version = "0.1.0"
description = "Script echo test extension"
trust = "third_party"
[runtime]
kind = "script"
runner = "sandboxed_process"
command = "echo-script"
args = []
[[capabilities]]
id = "script.echo"
description = "Echo text through script runtime"
effects = ["dispatch_capability"]
default_permission = "allow"
parameters_schema = { type = "object" }
"#;

146
docs/reborn/harness/e2e.md Normal file
View File

@@ -0,0 +1,146 @@
# Reborn E2E Harness
This document is the branch-local map for the dedicated Reborn E2E gate. Reborn is not missing tests: `main` already contains extensive crate-level contract and integration coverage. The gap this branch closes is a single, named E2E workflow that runs the Reborn architecture spine together and keeps a small product-surface smoke check beside it.
## What already exists
| Area | Existing coverage |
| --- | --- |
| Architecture boundaries | `crates/ironclaw_architecture/tests/reborn_dependency_boundaries.rs` |
| Host runtime facade and outcomes | `crates/ironclaw_host_runtime/tests/host_runtime_contract.rs` |
| Host runtime production composition | `crates/ironclaw_host_runtime/tests/host_runtime_services_contract.rs` |
| Dedicated Reborn E2E spine | `crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs` |
| Capability host invoke/resume/spawn | `crates/ironclaw_capabilities/tests/capability_host_*` |
| Dispatcher adapter selection | `crates/ironclaw_dispatcher/tests/vertical_slice_contract.rs` |
| WASM runtime lane | `crates/ironclaw_wasm/tests/wasm_dispatch_integration.rs` and `wasm_http_adapter_contract.rs` |
| Script runtime lane | `crates/ironclaw_scripts/tests/script_dispatch_integration.rs` and `script_http_adapter_contract.rs` |
| MCP runtime lane | `crates/ironclaw_mcp/tests/mcp_dispatch_integration.rs` and `mcp_adapter_contract.rs` |
| Process lifecycle | `crates/ironclaw_processes/tests/process_dispatch_integration.rs` and process service/store contracts |
| Network policy and host HTTP egress | `crates/ironclaw_network/tests/*` plus host-runtime HTTP egress tests |
| Secret storage/leases | `crates/ironclaw_secrets/tests/secret_store_contract.rs` plus host-runtime staged-secret tests |
| Events/audit replay | `crates/ironclaw_events/tests/durable_log_contract.rs` and host-runtime durable-event tests |
| Gateway product smoke | Existing Playwright scenarios under `tests/e2e/scenarios/`, especially `test_v2_*` |
## Dedicated Reborn E2E goal
The dedicated gate should answer one question:
```text
Can the Reborn architecture path still execute happy, blocked, denied, failed,
background, network, secret, event, and product-smoke paths after a change?
```
It intentionally reuses existing deterministic contract/integration tests instead of duplicating them in a second test framework.
## Happy path spine
The Reborn E2E happy path is covered by a dedicated `reborn_e2e_gate.rs` spine test plus the broader host-runtime and runtime-lane tests:
```text
Extension manifests
-> ExtensionRegistry
-> HostRuntimeServices
-> DefaultHostRuntime / HostRuntime facade
-> CapabilityHost authorization and run-state lifecycle
-> RuntimeDispatcher adapter selection
-> WASM / Script / MCP runtime adapters
-> resource reservation and reconciliation
-> durable runtime events
-> structured outcome returned to caller
```
Required assertions across the suite:
- visible capability surfaces include expected descriptors and stable surface version;
- health reports missing runtime backends fail-closed and configured backends ready;
- authorized invocations reach the selected runtime adapter;
- resource reservations are reconciled or released;
- run-state reaches the expected terminal or blocked state;
- durable events are replayable and metadata-only;
- runtime output is structured JSON and redacted where obligations require it.
## Other paths
The dedicated gate includes new Reborn E2E gate tests plus existing tests for these non-happy paths:
### Authorization and approval
- denied authorization fails before dispatch;
- approval-required invocation blocks with a persisted approval request id;
- approved resume consumes the exact lease once;
- changed input, wrong scope/user, expired lease, or missing stores fail before dispatch;
- unsupported obligations fail closed.
### Runtime availability and adapter failures
- missing runtime backend reports a stable missing-runtime failure;
- runtime lane errors map to stable failure categories;
- dispatcher/runtime adapter boundaries remain dependency-clean.
### Resource, process, and cancellation
- reservations are released on failure and reconciled on success;
- spawned background processes publish started/completed/failed/killed transitions;
- cancellation reaches the process graph;
- late completion after kill does not publish a misleading success;
- process handoffs for the same scoped capability fail closed while active.
### Network and secrets
- runtime HTTP egress is host-mediated;
- missing staged network policy fails before transport;
- staged secret material is consumed once;
- runtime-supplied manual credentials are rejected;
- raw secrets, credential-shaped values, and private host paths are not exposed in runtime-visible output, errors, events, or audit records.
### Event and observability
- durable event cursors replay runtime events;
- stale/gap cursor behavior is deterministic;
- event/audit records carry correlation metadata without raw payload leaks.
### Product smoke
A small Playwright smoke scenario starts an isolated `ENGINE_V2=true` gateway with the mock LLM and verifies:
- authenticated web shell loads;
- text-only chat completes and persists;
- tool-capable prompt completes through the gateway history path;
- no duplicate assistant response is emitted for a single user turn.
This smoke test does not replace the full browser E2E workflow. It proves the branch remains product-bootable while the Rust Reborn gate proves architecture behavior.
## Local commands
Run the deterministic Rust Reborn gate:
```bash
# Full gate
scripts/reborn-e2e-rust.sh
# Or run one CI matrix group at a time
scripts/reborn-e2e-rust.sh architecture
scripts/reborn-e2e-rust.sh runtimes
scripts/reborn-e2e-rust.sh substrates
```
The script expands to the dedicated `reborn_e2e_gate.rs` tests plus the current Reborn boundary, host-runtime, capability-host, dispatcher, WASM, Script, MCP, process, event, filesystem, network, secret, resource, run-state, approval, and authorization contract tests. Use the script as the source of truth for local/CI parity rather than copying individual `cargo test` commands.
Run the gateway smoke test:
```bash
cargo build --no-default-features --features libsql
cd tests/e2e
pip install -e .
playwright install --with-deps chromium # on Linux CI; local macOS can omit --with-deps
pytest scenarios/test_reborn_gateway_smoke.py -v --timeout=120
```
## CI ownership
`reborn-e2e.yml` is intentionally separate from `e2e.yml`:
- Reborn changes can run a focused architecture gate without destabilizing the main browser E2E matrix.
- The workflow is advisory by default and intentionally does not run on `merge_group`; add a merge-queue trigger only after the gate proves stable and is deliberately promoted to branch protection.
- The Rust jobs should stay deterministic and avoid live providers.
- The gateway job should remain a smoke test, not a second full browser matrix.

104
scripts/reborn-e2e-rust.sh Executable file
View File

@@ -0,0 +1,104 @@
#!/usr/bin/env bash
set -euo pipefail
# Run the deterministic Rust-side Reborn E2E gate.
# Usage:
# scripts/reborn-e2e-rust.sh # all groups
# scripts/reborn-e2e-rust.sh architecture # boundary + host runtime spine
# scripts/reborn-e2e-rust.sh runtimes # dispatcher/runtime/process lanes
# scripts/reborn-e2e-rust.sh substrates # event/network/secret substrates
#
# Extra cargo test args can be passed through CARGO_TEST_ARGS, for example:
# CARGO_TEST_ARGS='-- --nocapture' scripts/reborn-e2e-rust.sh architecture
group="${1:-all}"
extra_args=${CARGO_TEST_ARGS:-"-- --nocapture"}
run_test() {
local package="$1"
local test_name="$2"
echo "::group::cargo test -p ${package} --test ${test_name}"
# shellcheck disable=SC2086 # extra_args intentionally expands into cargo's trailing args.
cargo test -p "${package}" --test "${test_name}" ${extra_args}
echo "::endgroup::"
}
run_architecture() {
run_test ironclaw_architecture reborn_dependency_boundaries
run_test ironclaw_host_runtime host_runtime_contract
run_test ironclaw_host_runtime host_runtime_services_contract
run_test ironclaw_host_runtime reborn_e2e_gate
run_test ironclaw_host_runtime reborn_invoke_vertical_slice
run_test ironclaw_host_runtime runtime_http_egress_contract
run_test ironclaw_host_runtime builtin_obligation_handler_contract
run_test ironclaw_host_runtime obligation_services_composition_contract
run_test ironclaw_host_runtime production_trust_contract
run_test ironclaw_capabilities capability_boundary_contract
run_test ironclaw_capabilities capability_host_contract
run_test ironclaw_capabilities capability_host_dispatcher_integration
run_test ironclaw_capabilities capability_host_process_integration
run_test ironclaw_capabilities capability_host_run_state_contract
run_test ironclaw_capabilities capability_host_spawn_contract
run_test ironclaw_capabilities capability_obligation_handler_contract
}
run_runtimes() {
run_test ironclaw_dispatcher boundary_contract
run_test ironclaw_dispatcher dispatch_contract
run_test ironclaw_dispatcher event_dispatch_contract
run_test ironclaw_dispatcher runtime_dispatcher_integration
run_test ironclaw_dispatcher vertical_slice_contract
run_test ironclaw_wasm wasm_dispatch_integration
run_test ironclaw_wasm wasm_http_adapter_contract
run_test ironclaw_wasm wit_tool_runtime_contract
run_test ironclaw_scripts script_dispatch_integration
run_test ironclaw_scripts script_http_adapter_contract
run_test ironclaw_scripts script_runner_contract
run_test ironclaw_mcp mcp_adapter_contract
run_test ironclaw_mcp mcp_dispatch_integration
run_test ironclaw_processes process_dispatch_integration
run_test ironclaw_processes process_host_contract
run_test ironclaw_processes process_services_contract
run_test ironclaw_processes process_store_contract
}
run_substrates() {
run_test ironclaw_events durable_log_contract
run_test ironclaw_filesystem catalog_contract
run_test ironclaw_filesystem filesystem_contract
run_test ironclaw_network boundary_contract
run_test ironclaw_network network_http_egress_contract
run_test ironclaw_network network_policy_contract
run_test ironclaw_secrets boundary_contract
run_test ironclaw_secrets secret_store_contract
run_test ironclaw_resources resource_governor_contract
run_test ironclaw_run_state approval_resolution_contract
run_test ironclaw_run_state run_state_contract
run_test ironclaw_approvals approval_resolution_contract
run_test ironclaw_approvals boundary_contract
run_test ironclaw_authorization boundary_contract
run_test ironclaw_authorization capability_access_contract
run_test ironclaw_authorization capability_lease_contract
}
case "${group}" in
architecture)
run_architecture
;;
runtimes)
run_runtimes
;;
substrates)
run_substrates
;;
all)
run_architecture
run_runtimes
run_substrates
;;
*)
echo "unknown Reborn E2E group: ${group}" >&2
echo "expected one of: architecture, runtimes, substrates, all" >&2
exit 2
;;
esac

View File

@@ -0,0 +1,265 @@
"""Dedicated Reborn gateway smoke E2E.
This is intentionally small. The Rust Reborn gate proves the host/runtime
architecture. This Playwright/API smoke test proves the reborn-main branch still
boots an isolated ENGINE_V2 gateway, serves the browser shell, persists a normal
chat turn, and completes a simple tool-capable turn without duplicate terminal
assistant responses.
"""
import asyncio
import os
import signal
import socket
import tempfile
import pytest
from playwright.async_api import expect
from helpers import AUTH_TOKEN, SEL, api_get, api_post, wait_for_ready
_REBORN_GATEWAY_DB_TMPDIR = tempfile.TemporaryDirectory(prefix="ironclaw-reborn-gateway-e2e-")
_REBORN_GATEWAY_HOME_TMPDIR = tempfile.TemporaryDirectory(prefix="ironclaw-reborn-gateway-e2e-home-")
def _reserve_loopback_sockets(count: int) -> list[socket.socket]:
sockets: list[socket.socket] = []
try:
while len(sockets) < count:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("127.0.0.1", 0))
sockets.append(sock)
return sockets
except Exception:
for sock in sockets:
sock.close()
raise
def _close_reserved_sockets(sockets: list[socket.socket]) -> None:
for sock in sockets:
if sock.fileno() != -1:
sock.close()
def _forward_coverage_env(env: dict[str, str]) -> None:
for key, value in os.environ.items():
if key.startswith(("CARGO_LLVM_COV", "LLVM_")) or key in {
"CARGO_ENCODED_RUSTFLAGS",
"CARGO_INCREMENTAL",
}:
env[key] = value
async def _stop_process(proc, *, sig=signal.SIGINT, timeout: float = 10) -> None:
async def _drain_pipes() -> None:
try:
await asyncio.wait_for(proc.communicate(), timeout=1)
except (asyncio.TimeoutError, ValueError):
pass
if proc.returncode is not None:
await _drain_pipes()
return
try:
proc.send_signal(sig)
except ProcessLookupError:
await _drain_pipes()
return
try:
await asyncio.wait_for(proc.wait(), timeout=timeout)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
await _drain_pipes()
@pytest.fixture(scope="module")
async def reborn_gateway_server(ironclaw_binary, mock_llm_server):
"""Start an isolated gateway configured for the Reborn/V2 product shell."""
home_dir = _REBORN_GATEWAY_HOME_TMPDIR.name
base_dir = os.path.join(home_dir, ".ironclaw")
os.makedirs(base_dir, exist_ok=True)
reserved_sockets = _reserve_loopback_sockets(2)
gateway_port = reserved_sockets[0].getsockname()[1]
http_port = reserved_sockets[1].getsockname()[1]
env = {
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
"HOME": home_dir,
"IRONCLAW_BASE_DIR": base_dir,
"RUST_LOG": "ironclaw=info",
"RUST_BACKTRACE": "1",
"ENGINE_V2": "true",
"AGENT_AUTO_APPROVE_TOOLS": "true",
"GATEWAY_ENABLED": "true",
"GATEWAY_HOST": "127.0.0.1",
"GATEWAY_PORT": str(gateway_port),
"GATEWAY_AUTH_TOKEN": AUTH_TOKEN,
"GATEWAY_USER_ID": "reborn-gateway-e2e-user",
"HTTP_HOST": "127.0.0.1",
"HTTP_PORT": str(http_port),
"CLI_ENABLED": "false",
"LLM_BACKEND": "openai_compatible",
"LLM_BASE_URL": mock_llm_server,
"LLM_API_KEY": "mock-api-key",
"LLM_MODEL": "mock-model",
"DATABASE_BACKEND": "libsql",
"LIBSQL_PATH": os.path.join(_REBORN_GATEWAY_DB_TMPDIR.name, "reborn-gateway-e2e.db"),
"SANDBOX_ENABLED": "false",
"SKILLS_ENABLED": "false",
"ROUTINES_ENABLED": "false",
"HEARTBEAT_ENABLED": "false",
"EMBEDDING_ENABLED": "false",
"WASM_ENABLED": "false",
"ONBOARD_COMPLETED": "true",
}
_forward_coverage_env(env)
_close_reserved_sockets(reserved_sockets)
proc = await asyncio.create_subprocess_exec(
ironclaw_binary,
"--no-onboard",
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env=env,
)
base_url = f"http://127.0.0.1:{gateway_port}"
try:
await wait_for_ready(f"{base_url}/api/health", timeout=60)
yield base_url
except TimeoutError:
if proc.returncode is None:
await _stop_process(proc, timeout=2)
stderr_text = ""
if proc.stderr:
try:
stderr_bytes = await asyncio.wait_for(proc.stderr.read(8192), timeout=2)
stderr_text = stderr_bytes.decode("utf-8", errors="replace")
except asyncio.TimeoutError:
pass
pytest.fail(
f"Reborn gateway smoke server failed to start on port {gateway_port}.\n"
f"stderr:\n{stderr_text}"
)
finally:
if proc.returncode is None:
await _stop_process(proc, sig=signal.SIGINT, timeout=10)
if proc.returncode is None:
await _stop_process(proc, sig=signal.SIGTERM, timeout=5)
@pytest.fixture
async def reborn_gateway_page(reborn_gateway_server, browser):
context = await browser.new_context(viewport={"width": 1280, "height": 720})
page = await context.new_page()
await page.goto(f"{reborn_gateway_server}/?token={AUTH_TOKEN}")
await page.wait_for_selector(SEL["auth_screen"], state="hidden", timeout=15000)
await page.wait_for_function(
"() => typeof sseHasConnectedBefore !== 'undefined' && sseHasConnectedBefore === true",
timeout=10000,
)
yield page
await context.close()
async def _create_thread(base_url: str) -> str:
response = await api_post(base_url, "/api/chat/thread/new", timeout=15)
response.raise_for_status()
return response.json()["id"]
async def _send_message(base_url: str, thread_id: str, content: str) -> None:
response = await api_post(
base_url,
"/api/chat/send",
json={"content": content, "thread_id": thread_id},
timeout=30,
)
assert response.status_code in (200, 202), response.text
async def _wait_for_terminal_turn(
base_url: str,
thread_id: str,
expected_user_input: str,
*,
timeout: float = 45.0,
) -> dict:
last_history = {}
for _ in range(int(timeout * 2)):
response = await api_get(
base_url,
f"/api/chat/history?thread_id={thread_id}",
timeout=15,
)
response.raise_for_status()
history = response.json()
last_history = history
turns = history.get("turns", [])
matching_turns = [
turn
for turn in turns
if expected_user_input in (turn.get("user_input") or "")
]
if matching_turns and (matching_turns[-1].get("response") or "").strip():
return matching_turns[-1]
await asyncio.sleep(0.5)
raise AssertionError(
f"Timed out waiting for terminal turn containing {expected_user_input!r}. "
f"Last history: {last_history}"
)
async def test_reborn_gateway_loads_engine_v2_shell(reborn_gateway_page):
"""The isolated Reborn smoke gateway should boot the ENGINE_V2 shell."""
chat_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="chat"))
missions_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="missions"))
routines_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="routines"))
await expect(chat_tab).to_be_visible()
await expect(missions_tab).to_be_visible()
await expect(routines_tab).to_be_hidden()
async def test_reborn_gateway_persists_text_and_tool_turns_without_duplicate_response(
reborn_gateway_server,
):
"""A text turn and an auto-approved tool turn should each produce one terminal response."""
thread_id = await _create_thread(reborn_gateway_server)
text_prompt = "reborn gateway smoke: what is 2+2?"
await _send_message(reborn_gateway_server, thread_id, text_prompt)
text_turn = await _wait_for_terminal_turn(reborn_gateway_server, thread_id, text_prompt)
assert "4" in text_turn.get("response", "")
tool_prompt = "echo reborn gateway smoke tool result"
await _send_message(reborn_gateway_server, thread_id, tool_prompt)
tool_turn = await _wait_for_terminal_turn(reborn_gateway_server, thread_id, tool_prompt)
assert "reborn gateway smoke tool result" in tool_turn.get("response", "").lower()
tool_calls = tool_turn.get("tool_calls", [])
assert tool_calls, f"Expected persisted tool call metadata, got: {tool_turn}"
assert any(call.get("name") == "echo" and call.get("has_result") for call in tool_calls)
history_response = await api_get(
reborn_gateway_server,
f"/api/chat/history?thread_id={thread_id}",
timeout=15,
)
history_response.raise_for_status()
matching_tool_turns = [
turn
for turn in history_response.json().get("turns", [])
if tool_prompt in (turn.get("user_input") or "")
and (turn.get("response") or "").strip()
]
assert len(matching_tool_turns) == 1, (
"Expected one terminal assistant response for the tool prompt, got "
f"{len(matching_tool_turns)} turns: {matching_tool_turns}"
)