mirror of
https://github.com/nearai/ironclaw.git
synced 2026-09-03 08:06:01 +08:00
test(reborn): add dedicated e2e gate (#3251)
This commit is contained in:
140
.github/workflows/reborn-e2e.yml
vendored
Normal file
140
.github/workflows/reborn-e2e.yml
vendored
Normal file
@@ -0,0 +1,140 @@
|
||||
name: Reborn E2E
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: Commit SHA or ref to test
|
||||
required: false
|
||||
type: string
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "crates/ironclaw_*/**"
|
||||
- "docs/reborn/**"
|
||||
- "scripts/reborn-e2e-rust.sh"
|
||||
- "tests/e2e/**"
|
||||
- "build.rs"
|
||||
- "providers.json"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- ".github/workflows/reborn-e2e.yml"
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "crates/ironclaw_*/**"
|
||||
- "docs/reborn/**"
|
||||
- "scripts/reborn-e2e-rust.sh"
|
||||
- "tests/e2e/**"
|
||||
- "build.rs"
|
||||
- "providers.json"
|
||||
- "Cargo.toml"
|
||||
- "Cargo.lock"
|
||||
- ".github/workflows/reborn-e2e.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: reborn-e2e-${{ github.event_name }}-${{ github.head_ref || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
rust-reborn:
|
||||
name: Rust Reborn (${{ matrix.group }})
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 35
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
group:
|
||||
- architecture
|
||||
- runtimes
|
||||
- substrates
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
|
||||
- name: Restore Rust cache
|
||||
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
|
||||
with:
|
||||
key: reborn-e2e-${{ matrix.group }}
|
||||
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Run deterministic Reborn Rust gate
|
||||
env:
|
||||
CARGO_TEST_ARGS: "-- --nocapture"
|
||||
run: scripts/reborn-e2e-rust.sh ${{ matrix.group }}
|
||||
|
||||
gateway-smoke:
|
||||
name: Reborn gateway smoke
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
with:
|
||||
ref: ${{ inputs.ref || github.sha }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install Rust
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
|
||||
- name: Restore Rust cache
|
||||
uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
|
||||
with:
|
||||
key: reborn-e2e-gateway-smoke
|
||||
save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Build ironclaw with libSQL
|
||||
run: cargo build --no-default-features --features libsql
|
||||
|
||||
- name: Install Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install E2E dependencies
|
||||
run: |
|
||||
cd tests/e2e
|
||||
pip install -e .
|
||||
playwright install --with-deps chromium
|
||||
|
||||
- name: Run Reborn gateway smoke
|
||||
run: pytest tests/e2e/scenarios/test_reborn_gateway_smoke.py -v --timeout=120
|
||||
|
||||
- name: Upload screenshots on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
with:
|
||||
name: reborn-e2e-screenshots
|
||||
path: tests/e2e/screenshots/
|
||||
if-no-files-found: ignore
|
||||
|
||||
reborn-e2e:
|
||||
name: Reborn E2E
|
||||
runs-on: ubuntu-latest
|
||||
if: always()
|
||||
needs:
|
||||
- rust-reborn
|
||||
- gateway-smoke
|
||||
steps:
|
||||
- name: Check Reborn E2E jobs
|
||||
run: |
|
||||
if [[ "${{ needs.rust-reborn.result }}" != "success" ]]; then
|
||||
echo "Rust Reborn E2E failed"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${{ needs.gateway-smoke.result }}" != "success" ]]; then
|
||||
echo "Reborn gateway smoke failed"
|
||||
exit 1
|
||||
fi
|
||||
735
crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs
Normal file
735
crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs
Normal file
@@ -0,0 +1,735 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
use ironclaw_approvals::LeaseApproval;
|
||||
use ironclaw_authorization::{
|
||||
CapabilityLeaseStatus, CapabilityLeaseStore, GrantAuthorizer, InMemoryCapabilityLeaseStore,
|
||||
TrustAwareCapabilityDispatchAuthorizer,
|
||||
};
|
||||
use ironclaw_events::{
|
||||
DurableEventLog, EventStreamKey, InMemoryDurableEventLog, InMemoryEventSink, ReadScope,
|
||||
RuntimeEventKind,
|
||||
};
|
||||
use ironclaw_extensions::{ExtensionManifest, ExtensionPackage, ExtensionRegistry};
|
||||
use ironclaw_filesystem::LocalFilesystem;
|
||||
use ironclaw_host_api::*;
|
||||
use ironclaw_host_runtime::{
|
||||
CapabilitySurfaceVersion, HostHttpEgressService, HostRuntime, HostRuntimeServices,
|
||||
NetworkObligationPolicyStore, RuntimeCapabilityOutcome, RuntimeCapabilityRequest,
|
||||
RuntimeCapabilityResumeRequest, RuntimeFailureKind, RuntimeSecretInjectionStore,
|
||||
RuntimeStatusRequest, SurfaceKind,
|
||||
};
|
||||
use ironclaw_network::{
|
||||
NetworkHttpEgress, NetworkHttpError, NetworkHttpRequest, NetworkHttpResponse, NetworkUsage,
|
||||
};
|
||||
use ironclaw_processes::{InMemoryProcessResultStore, InMemoryProcessStore, ProcessServices};
|
||||
use ironclaw_resources::{InMemoryResourceGovernor, ResourceAccount, ResourceTally};
|
||||
use ironclaw_run_state::{
|
||||
InMemoryApprovalRequestStore, InMemoryRunStateStore, RunStateStore, RunStatus,
|
||||
};
|
||||
use ironclaw_scripts::{
|
||||
ScriptBackend, ScriptBackendOutput, ScriptBackendRequest, ScriptRuntime, ScriptRuntimeConfig,
|
||||
};
|
||||
use ironclaw_secrets::{InMemorySecretStore, SecretMaterial};
|
||||
use ironclaw_trust::{
|
||||
AdminConfig, AdminEntry, AuthorityCeiling, EffectiveTrustClass, HostTrustAssignment,
|
||||
HostTrustPolicy, TrustDecision, TrustProvenance,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[tokio::test]
|
||||
async fn reborn_e2e_gate_invokes_script_through_host_runtime_with_status_events_and_resources() {
|
||||
let governor = Arc::new(InMemoryResourceGovernor::new());
|
||||
let run_state = Arc::new(InMemoryRunStateStore::new());
|
||||
let event_log = Arc::new(InMemoryDurableEventLog::new());
|
||||
let services = HostRuntimeServices::new(
|
||||
Arc::new(registry_with_manifest(SCRIPT_MANIFEST)),
|
||||
Arc::new(LocalFilesystem::new()),
|
||||
Arc::clone(&governor),
|
||||
Arc::new(GrantAuthorizer::new()),
|
||||
ProcessServices::in_memory(),
|
||||
CapabilitySurfaceVersion::new("surface-v1").unwrap(),
|
||||
)
|
||||
.with_trust_policy(Arc::new(local_manifest_trust_policy()))
|
||||
.with_run_state(Arc::clone(&run_state))
|
||||
.with_script_runtime(Arc::new(ScriptRuntime::new(
|
||||
ScriptRuntimeConfig::for_testing(),
|
||||
EchoScriptBackend,
|
||||
)))
|
||||
.with_durable_event_log(Arc::clone(&event_log));
|
||||
let runtime = services.host_runtime();
|
||||
let context = execution_context_with_dispatch_grant();
|
||||
let scope = context.resource_scope.clone();
|
||||
let invocation_id = context.invocation_id;
|
||||
|
||||
let surface = runtime
|
||||
.visible_capabilities(ironclaw_host_runtime::VisibleCapabilityRequest::new(
|
||||
scope.clone(),
|
||||
context.correlation_id,
|
||||
SurfaceKind::new("gateway-smoke").unwrap(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(surface.version.as_str(), "surface-v1");
|
||||
assert_eq!(surface.descriptors.len(), 1);
|
||||
assert_eq!(surface.descriptors[0].id, script_capability_id());
|
||||
|
||||
let health = runtime.health().await.unwrap();
|
||||
assert!(health.ready);
|
||||
assert!(health.missing_runtime_backends.is_empty());
|
||||
|
||||
let status_before = runtime
|
||||
.runtime_status(RuntimeStatusRequest::new(
|
||||
scope.clone(),
|
||||
CorrelationId::new(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(status_before.active_work.is_empty());
|
||||
|
||||
let input = json!({
|
||||
"message": "reborn e2e happy path",
|
||||
"secret_sentinel": "SECRET_REBORN_E2E_GATE_SHOULD_NOT_LEAK",
|
||||
"host_path_sentinel": "/private/tmp/reborn-e2e-gate"
|
||||
});
|
||||
let outcome = runtime
|
||||
.invoke_capability(RuntimeCapabilityRequest::new(
|
||||
context,
|
||||
script_capability_id(),
|
||||
ResourceEstimate {
|
||||
output_bytes: Some(4096),
|
||||
..ResourceEstimate::default()
|
||||
},
|
||||
input.clone(),
|
||||
trust_decision_with_dispatch_authority(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
match outcome {
|
||||
RuntimeCapabilityOutcome::Completed(completed) => {
|
||||
assert_eq!(completed.capability_id, script_capability_id());
|
||||
assert_eq!(completed.output, input);
|
||||
assert!(completed.usage.output_bytes > 0);
|
||||
}
|
||||
other => panic!("expected completed outcome, got {other:?}"),
|
||||
}
|
||||
|
||||
let run = run_state.get(&scope, invocation_id).await.unwrap().unwrap();
|
||||
assert_eq!(run.status, RunStatus::Completed);
|
||||
let tenant_account = ResourceAccount::tenant(scope.tenant_id.clone());
|
||||
assert_eq!(
|
||||
governor.reserved_for(&tenant_account),
|
||||
ResourceTally::default()
|
||||
);
|
||||
assert!(governor.usage_for(&tenant_account).output_bytes > 0);
|
||||
|
||||
let status_after = runtime
|
||||
.runtime_status(RuntimeStatusRequest::new(
|
||||
scope.clone(),
|
||||
CorrelationId::new(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(status_after.active_work.is_empty());
|
||||
|
||||
let replay = event_log
|
||||
.read_after_cursor(
|
||||
&EventStreamKey::from_scope(&scope),
|
||||
&ReadScope::any(),
|
||||
None,
|
||||
10,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let kinds = replay
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| entry.record.kind)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
kinds,
|
||||
vec![
|
||||
RuntimeEventKind::DispatchRequested,
|
||||
RuntimeEventKind::RuntimeSelected,
|
||||
RuntimeEventKind::DispatchSucceeded,
|
||||
]
|
||||
);
|
||||
let serialized = serde_json::to_string(&replay).unwrap();
|
||||
for forbidden in [
|
||||
"SECRET_REBORN_E2E_GATE_SHOULD_NOT_LEAK",
|
||||
"/private/tmp/reborn-e2e-gate",
|
||||
] {
|
||||
assert!(
|
||||
!serialized.contains(forbidden),
|
||||
"durable Reborn E2E event replay leaked {forbidden}: {serialized}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reborn_e2e_gate_blocks_for_approval_resumes_once_and_rejects_replay() {
|
||||
let fixture = approval_resume_fixture();
|
||||
let runtime = fixture.services.host_runtime();
|
||||
let context = execution_context_without_grants();
|
||||
let scope = context.resource_scope.clone();
|
||||
let invocation_id = context.invocation_id;
|
||||
let input = json!({"message": "approval resume through Reborn E2E gate"});
|
||||
|
||||
let gate = block_for_approval(&runtime, context.clone(), input.clone()).await;
|
||||
let blocked_run = fixture
|
||||
.run_state
|
||||
.get(&scope, invocation_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(blocked_run.status, RunStatus::BlockedApproval);
|
||||
assert_eq!(
|
||||
blocked_run.approval_request_id,
|
||||
Some(gate.approval_request_id)
|
||||
);
|
||||
|
||||
let lease =
|
||||
approve_dispatch_for_services(&fixture.services, &scope, gate.approval_request_id).await;
|
||||
|
||||
let resumed = runtime
|
||||
.resume_capability(RuntimeCapabilityResumeRequest::new(
|
||||
context.clone(),
|
||||
gate.approval_request_id,
|
||||
script_capability_id(),
|
||||
ResourceEstimate::default(),
|
||||
input.clone(),
|
||||
trust_decision_with_dispatch_authority(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
match resumed {
|
||||
RuntimeCapabilityOutcome::Completed(completed) => {
|
||||
assert_eq!(completed.capability_id, script_capability_id());
|
||||
assert_eq!(completed.output, input);
|
||||
}
|
||||
other => panic!("expected completed approval resume, got {other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
fixture
|
||||
.capability_leases
|
||||
.get(&scope, lease.grant.id)
|
||||
.await
|
||||
.unwrap()
|
||||
.status,
|
||||
CapabilityLeaseStatus::Consumed
|
||||
);
|
||||
assert_event_kinds(
|
||||
&fixture.events,
|
||||
&[
|
||||
RuntimeEventKind::DispatchRequested,
|
||||
RuntimeEventKind::RuntimeSelected,
|
||||
RuntimeEventKind::DispatchSucceeded,
|
||||
],
|
||||
);
|
||||
|
||||
let replay = runtime
|
||||
.resume_capability(RuntimeCapabilityResumeRequest::new(
|
||||
context,
|
||||
gate.approval_request_id,
|
||||
script_capability_id(),
|
||||
ResourceEstimate::default(),
|
||||
json!({"message": "approval resume through Reborn E2E gate"}),
|
||||
trust_decision_with_dispatch_authority(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_failed_outcome(replay, RuntimeFailureKind::Authorization);
|
||||
assert_eq!(
|
||||
fixture.events.events().len(),
|
||||
3,
|
||||
"replayed approval resume must fail before a second runtime dispatch"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reborn_e2e_gate_fails_unsupported_obligations_before_runtime_events_or_success() {
|
||||
let run_state = Arc::new(InMemoryRunStateStore::new());
|
||||
let events = InMemoryEventSink::new();
|
||||
let governor = Arc::new(InMemoryResourceGovernor::new());
|
||||
let services = HostRuntimeServices::new(
|
||||
Arc::new(registry_with_manifest(SCRIPT_MANIFEST)),
|
||||
Arc::new(LocalFilesystem::new()),
|
||||
Arc::clone(&governor),
|
||||
Arc::new(ObligatingAuthorizer),
|
||||
ProcessServices::in_memory(),
|
||||
CapabilitySurfaceVersion::new("surface-v1").unwrap(),
|
||||
)
|
||||
.with_trust_policy(Arc::new(local_manifest_trust_policy()))
|
||||
.with_run_state(Arc::clone(&run_state))
|
||||
.with_script_runtime(Arc::new(ScriptRuntime::new(
|
||||
ScriptRuntimeConfig::for_testing(),
|
||||
EchoScriptBackend,
|
||||
)))
|
||||
.with_event_sink(Arc::new(events.clone()));
|
||||
let runtime = services.host_runtime();
|
||||
let context = execution_context_with_dispatch_grant();
|
||||
let scope = context.resource_scope.clone();
|
||||
let invocation_id = context.invocation_id;
|
||||
|
||||
let outcome = runtime
|
||||
.invoke_capability(RuntimeCapabilityRequest::new(
|
||||
context,
|
||||
script_capability_id(),
|
||||
ResourceEstimate::default(),
|
||||
json!({"message": "unsupported obligation"}),
|
||||
trust_decision_with_dispatch_authority(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_failed_outcome(outcome, RuntimeFailureKind::Backend);
|
||||
assert!(events.events().is_empty());
|
||||
let run = run_state.get(&scope, invocation_id).await.unwrap().unwrap();
|
||||
assert_eq!(run.status, RunStatus::Failed);
|
||||
assert_eq!(run.error_kind.as_deref(), Some("ObligationFailed"));
|
||||
let tenant_account = ResourceAccount::tenant(scope.tenant_id.clone());
|
||||
assert_eq!(
|
||||
governor.reserved_for(&tenant_account),
|
||||
ResourceTally::default()
|
||||
);
|
||||
assert_eq!(
|
||||
governor.usage_for(&tenant_account),
|
||||
ResourceTally::default()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reborn_e2e_gate_host_http_consumes_staged_policy_and_secret_once() {
|
||||
let network = RecordingNetwork::ok(NetworkHttpResponse {
|
||||
status: 200,
|
||||
headers: vec![],
|
||||
body: br#"{"ok":true}"#.to_vec(),
|
||||
usage: NetworkUsage {
|
||||
request_bytes: 5,
|
||||
response_bytes: 11,
|
||||
resolved_ip: None,
|
||||
},
|
||||
});
|
||||
let network_recorder = network.requests.clone();
|
||||
let policy_store = Arc::new(NetworkObligationPolicyStore::new());
|
||||
let secret_injections = Arc::new(RuntimeSecretInjectionStore::new());
|
||||
let scope = sample_scope(InvocationId::new());
|
||||
let capability_id = script_capability_id();
|
||||
let handle = SecretHandle::new("api-token").unwrap();
|
||||
let staged_policy = sample_policy();
|
||||
policy_store.insert(&scope, &capability_id, staged_policy.clone());
|
||||
secret_injections
|
||||
.insert(
|
||||
&scope,
|
||||
&capability_id,
|
||||
&handle,
|
||||
SecretMaterial::from("sk-reborn-e2e-staged-secret"),
|
||||
)
|
||||
.unwrap();
|
||||
let service = HostHttpEgressService::new(network, InMemorySecretStore::new())
|
||||
.with_network_policy_store(Arc::clone(&policy_store))
|
||||
.with_secret_injection_store(Arc::clone(&secret_injections));
|
||||
|
||||
let request = RuntimeHttpEgressRequest {
|
||||
runtime: RuntimeKind::Script,
|
||||
scope: scope.clone(),
|
||||
capability_id: capability_id.clone(),
|
||||
method: NetworkMethod::Post,
|
||||
url: "https://api.example.test/v1/run".to_string(),
|
||||
headers: vec![],
|
||||
body: b"hello".to_vec(),
|
||||
network_policy: caller_supplied_policy(),
|
||||
credential_injections: vec![RuntimeCredentialInjection {
|
||||
handle: handle.clone(),
|
||||
source: RuntimeCredentialSource::StagedObligation {
|
||||
capability_id: capability_id.clone(),
|
||||
},
|
||||
target: RuntimeCredentialTarget::Header {
|
||||
name: "authorization".to_string(),
|
||||
prefix: Some("Bearer ".to_string()),
|
||||
},
|
||||
required: true,
|
||||
}],
|
||||
response_body_limit: Some(4096),
|
||||
timeout_ms: None,
|
||||
};
|
||||
|
||||
let response = service
|
||||
.execute(request.clone())
|
||||
.expect("host HTTP egress should use staged Reborn policy and secret material");
|
||||
assert_eq!(response.status, 200);
|
||||
let recorded = network_recorder.lock().unwrap();
|
||||
assert_eq!(recorded.len(), 1);
|
||||
assert_eq!(recorded[0].policy, staged_policy);
|
||||
assert_eq!(
|
||||
recorded[0]
|
||||
.headers
|
||||
.iter()
|
||||
.find(|(name, _)| name == "authorization"),
|
||||
Some(&(
|
||||
"authorization".to_string(),
|
||||
"Bearer sk-reborn-e2e-staged-secret".to_string()
|
||||
))
|
||||
);
|
||||
drop(recorded);
|
||||
assert!(
|
||||
secret_injections
|
||||
.take(&scope, &capability_id, &handle)
|
||||
.unwrap()
|
||||
.is_none(),
|
||||
"staged secret material must be consumed exactly once"
|
||||
);
|
||||
assert_eq!(
|
||||
policy_store.get(&scope, &capability_id),
|
||||
Some(staged_policy),
|
||||
"host egress must leave staged network policy for invocation/process lifecycle cleanup"
|
||||
);
|
||||
|
||||
let replay = service
|
||||
.execute(request)
|
||||
.expect_err("consumed staged secret must not be reusable");
|
||||
assert!(matches!(replay, RuntimeHttpEgressError::Credential { .. }));
|
||||
assert_eq!(
|
||||
network_recorder.lock().unwrap().len(),
|
||||
1,
|
||||
"replay must fail before a second outbound transport attempt"
|
||||
);
|
||||
}
|
||||
|
||||
type InMemoryServices = HostRuntimeServices<
|
||||
LocalFilesystem,
|
||||
InMemoryResourceGovernor,
|
||||
InMemoryProcessStore,
|
||||
InMemoryProcessResultStore,
|
||||
>;
|
||||
|
||||
struct ApprovalFixture {
|
||||
services: InMemoryServices,
|
||||
run_state: Arc<InMemoryRunStateStore>,
|
||||
capability_leases: Arc<InMemoryCapabilityLeaseStore>,
|
||||
events: InMemoryEventSink,
|
||||
}
|
||||
|
||||
fn approval_resume_fixture() -> ApprovalFixture {
|
||||
let run_state = Arc::new(InMemoryRunStateStore::new());
|
||||
let approval_requests = Arc::new(InMemoryApprovalRequestStore::new());
|
||||
let capability_leases = Arc::new(InMemoryCapabilityLeaseStore::new());
|
||||
let events = InMemoryEventSink::new();
|
||||
let services = HostRuntimeServices::new(
|
||||
Arc::new(registry_with_manifest(SCRIPT_MANIFEST)),
|
||||
Arc::new(LocalFilesystem::new()),
|
||||
Arc::new(InMemoryResourceGovernor::new()),
|
||||
Arc::new(ApprovalThenGrantAuthorizer),
|
||||
ProcessServices::in_memory(),
|
||||
CapabilitySurfaceVersion::new("surface-v1").unwrap(),
|
||||
)
|
||||
.with_trust_policy(Arc::new(local_manifest_trust_policy()))
|
||||
.with_run_state(Arc::clone(&run_state))
|
||||
.with_approval_requests(approval_requests)
|
||||
.with_capability_leases(Arc::clone(&capability_leases))
|
||||
.with_script_runtime(Arc::new(ScriptRuntime::new(
|
||||
ScriptRuntimeConfig::for_testing(),
|
||||
EchoScriptBackend,
|
||||
)))
|
||||
.with_event_sink(Arc::new(events.clone()));
|
||||
|
||||
ApprovalFixture {
|
||||
services,
|
||||
run_state,
|
||||
capability_leases,
|
||||
events,
|
||||
}
|
||||
}
|
||||
|
||||
async fn block_for_approval(
|
||||
runtime: &impl HostRuntime,
|
||||
context: ExecutionContext,
|
||||
input: serde_json::Value,
|
||||
) -> ironclaw_host_runtime::RuntimeApprovalGate {
|
||||
let outcome = runtime
|
||||
.invoke_capability(RuntimeCapabilityRequest::new(
|
||||
context,
|
||||
script_capability_id(),
|
||||
ResourceEstimate::default(),
|
||||
input,
|
||||
trust_decision_with_dispatch_authority(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
match outcome {
|
||||
RuntimeCapabilityOutcome::ApprovalRequired(gate) => gate,
|
||||
other => panic!("expected approval gate, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
async fn approve_dispatch_for_services(
|
||||
services: &InMemoryServices,
|
||||
scope: &ResourceScope,
|
||||
approval_request_id: ApprovalRequestId,
|
||||
) -> ironclaw_authorization::CapabilityLease {
|
||||
services
|
||||
.approval_resolver()
|
||||
.expect("approval resolver should be configured")
|
||||
.approve_dispatch(
|
||||
scope,
|
||||
approval_request_id,
|
||||
LeaseApproval {
|
||||
issued_by: Principal::HostRuntime,
|
||||
allowed_effects: vec![EffectKind::DispatchCapability],
|
||||
mounts: MountView::default(),
|
||||
network: NetworkPolicy::default(),
|
||||
secrets: Vec::new(),
|
||||
resource_ceiling: None,
|
||||
expires_at: None,
|
||||
max_invocations: Some(1),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
struct ApprovalThenGrantAuthorizer;
|
||||
|
||||
#[async_trait]
|
||||
impl TrustAwareCapabilityDispatchAuthorizer for ApprovalThenGrantAuthorizer {
|
||||
async fn authorize_dispatch_with_trust(
|
||||
&self,
|
||||
context: &ExecutionContext,
|
||||
descriptor: &CapabilityDescriptor,
|
||||
estimate: &ResourceEstimate,
|
||||
trust_decision: &TrustDecision,
|
||||
) -> Decision {
|
||||
if context.grants.grants.is_empty() {
|
||||
Decision::RequireApproval {
|
||||
request: ApprovalRequest {
|
||||
id: ApprovalRequestId::new(),
|
||||
correlation_id: context.correlation_id,
|
||||
requested_by: Principal::Extension(context.extension_id.clone()),
|
||||
action: Box::new(Action::Dispatch {
|
||||
capability: descriptor.id.clone(),
|
||||
estimated_resources: estimate.clone(),
|
||||
}),
|
||||
invocation_fingerprint: None,
|
||||
reason: "approval required".to_string(),
|
||||
reusable_scope: None,
|
||||
},
|
||||
}
|
||||
} else {
|
||||
GrantAuthorizer::new()
|
||||
.authorize_dispatch_with_trust(context, descriptor, estimate, trust_decision)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ObligatingAuthorizer;
|
||||
|
||||
#[async_trait]
|
||||
impl TrustAwareCapabilityDispatchAuthorizer for ObligatingAuthorizer {
|
||||
async fn authorize_dispatch_with_trust(
|
||||
&self,
|
||||
_context: &ExecutionContext,
|
||||
_descriptor: &CapabilityDescriptor,
|
||||
_estimate: &ResourceEstimate,
|
||||
_trust_decision: &TrustDecision,
|
||||
) -> Decision {
|
||||
Decision::Allow {
|
||||
obligations: Obligations::new(vec![Obligation::AuditBefore]).unwrap(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct EchoScriptBackend;
|
||||
|
||||
impl ScriptBackend for EchoScriptBackend {
|
||||
fn execute(&self, request: ScriptBackendRequest) -> Result<ScriptBackendOutput, String> {
|
||||
let value = serde_json::from_str(&request.stdin_json).map_err(|error| error.to_string())?;
|
||||
Ok(ScriptBackendOutput::json(value))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RecordingNetwork {
|
||||
response: Result<NetworkHttpResponse, NetworkHttpError>,
|
||||
requests: Arc<Mutex<Vec<NetworkHttpRequest>>>,
|
||||
}
|
||||
|
||||
impl RecordingNetwork {
|
||||
fn ok(response: NetworkHttpResponse) -> Self {
|
||||
Self {
|
||||
response: Ok(response),
|
||||
requests: Arc::new(Mutex::new(Vec::new())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl NetworkHttpEgress for RecordingNetwork {
|
||||
fn execute(
|
||||
&self,
|
||||
request: NetworkHttpRequest,
|
||||
) -> Result<NetworkHttpResponse, NetworkHttpError> {
|
||||
self.requests.lock().unwrap().push(request);
|
||||
self.response.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn registry_with_manifest(manifest: &str) -> ExtensionRegistry {
|
||||
let mut registry = ExtensionRegistry::new();
|
||||
let manifest = ExtensionManifest::parse(manifest).unwrap();
|
||||
let package = ExtensionPackage::from_manifest(
|
||||
manifest,
|
||||
VirtualPath::new("/system/extensions/script").unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
registry.insert(package).unwrap();
|
||||
registry
|
||||
}
|
||||
|
||||
fn execution_context_with_dispatch_grant() -> ExecutionContext {
|
||||
let mut grants = CapabilitySet::default();
|
||||
grants.grants.push(dispatch_grant());
|
||||
ExecutionContext::local_default(
|
||||
UserId::new("user").unwrap(),
|
||||
ExtensionId::new("caller").unwrap(),
|
||||
RuntimeKind::Script,
|
||||
TrustClass::UserTrusted,
|
||||
grants,
|
||||
MountView::default(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn execution_context_without_grants() -> ExecutionContext {
|
||||
ExecutionContext::local_default(
|
||||
UserId::new("user").unwrap(),
|
||||
ExtensionId::new("caller").unwrap(),
|
||||
RuntimeKind::Script,
|
||||
TrustClass::UserTrusted,
|
||||
CapabilitySet::default(),
|
||||
MountView::default(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn dispatch_grant() -> CapabilityGrant {
|
||||
CapabilityGrant {
|
||||
id: CapabilityGrantId::new(),
|
||||
capability: script_capability_id(),
|
||||
grantee: Principal::Extension(ExtensionId::new("caller").unwrap()),
|
||||
issued_by: Principal::HostRuntime,
|
||||
constraints: GrantConstraints {
|
||||
allowed_effects: vec![EffectKind::DispatchCapability],
|
||||
mounts: MountView::default(),
|
||||
network: NetworkPolicy::default(),
|
||||
secrets: Vec::new(),
|
||||
resource_ceiling: None,
|
||||
expires_at: None,
|
||||
max_invocations: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn local_manifest_trust_policy() -> HostTrustPolicy {
|
||||
HostTrustPolicy::new(vec![Box::new(AdminConfig::with_entries(vec![
|
||||
AdminEntry::for_local_manifest(
|
||||
PackageId::new("script").unwrap(),
|
||||
"/system/extensions/script/manifest.toml".to_string(),
|
||||
None,
|
||||
HostTrustAssignment::user_trusted(),
|
||||
vec![EffectKind::DispatchCapability],
|
||||
None,
|
||||
),
|
||||
]))])
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn trust_decision_with_dispatch_authority() -> TrustDecision {
|
||||
TrustDecision {
|
||||
effective_trust: EffectiveTrustClass::user_trusted(),
|
||||
authority_ceiling: AuthorityCeiling {
|
||||
allowed_effects: vec![EffectKind::DispatchCapability],
|
||||
max_resource_ceiling: None,
|
||||
},
|
||||
provenance: TrustProvenance::Default,
|
||||
evaluated_at: Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_scope(invocation_id: InvocationId) -> ResourceScope {
|
||||
ResourceScope {
|
||||
tenant_id: TenantId::new("tenant1").unwrap(),
|
||||
user_id: UserId::new("user1").unwrap(),
|
||||
agent_id: None,
|
||||
project_id: None,
|
||||
mission_id: None,
|
||||
thread_id: None,
|
||||
invocation_id,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_policy() -> NetworkPolicy {
|
||||
NetworkPolicy {
|
||||
allowed_targets: vec![NetworkTargetPattern {
|
||||
scheme: Some(NetworkScheme::Https),
|
||||
host_pattern: "api.example.test".to_string(),
|
||||
port: None,
|
||||
}],
|
||||
deny_private_ip_ranges: true,
|
||||
max_egress_bytes: Some(4096),
|
||||
}
|
||||
}
|
||||
|
||||
fn caller_supplied_policy() -> NetworkPolicy {
|
||||
NetworkPolicy {
|
||||
allowed_targets: vec![NetworkTargetPattern {
|
||||
scheme: Some(NetworkScheme::Https),
|
||||
host_pattern: "caller.example.test".to_string(),
|
||||
port: None,
|
||||
}],
|
||||
deny_private_ip_ranges: false,
|
||||
max_egress_bytes: Some(1),
|
||||
}
|
||||
}
|
||||
|
||||
fn script_capability_id() -> CapabilityId {
|
||||
CapabilityId::new("script.echo").unwrap()
|
||||
}
|
||||
|
||||
fn assert_event_kinds(events: &InMemoryEventSink, expected: &[RuntimeEventKind]) {
|
||||
let actual = events
|
||||
.events()
|
||||
.into_iter()
|
||||
.map(|event| event.kind)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
fn assert_failed_outcome(outcome: RuntimeCapabilityOutcome, expected: RuntimeFailureKind) {
|
||||
match outcome {
|
||||
RuntimeCapabilityOutcome::Failed(failure) => assert_eq!(failure.kind, expected),
|
||||
other => panic!("expected failed outcome {expected:?}, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
const SCRIPT_MANIFEST: &str = r#"
|
||||
id = "script"
|
||||
name = "Script Echo"
|
||||
version = "0.1.0"
|
||||
description = "Script echo test extension"
|
||||
trust = "third_party"
|
||||
|
||||
[runtime]
|
||||
kind = "script"
|
||||
runner = "sandboxed_process"
|
||||
command = "echo-script"
|
||||
args = []
|
||||
|
||||
[[capabilities]]
|
||||
id = "script.echo"
|
||||
description = "Echo text through script runtime"
|
||||
effects = ["dispatch_capability"]
|
||||
default_permission = "allow"
|
||||
parameters_schema = { type = "object" }
|
||||
"#;
|
||||
146
docs/reborn/harness/e2e.md
Normal file
146
docs/reborn/harness/e2e.md
Normal file
@@ -0,0 +1,146 @@
|
||||
# Reborn E2E Harness
|
||||
|
||||
This document is the branch-local map for the dedicated Reborn E2E gate. Reborn is not missing tests: `main` already contains extensive crate-level contract and integration coverage. The gap this branch closes is a single, named E2E workflow that runs the Reborn architecture spine together and keeps a small product-surface smoke check beside it.
|
||||
|
||||
## What already exists
|
||||
|
||||
| Area | Existing coverage |
|
||||
| --- | --- |
|
||||
| Architecture boundaries | `crates/ironclaw_architecture/tests/reborn_dependency_boundaries.rs` |
|
||||
| Host runtime facade and outcomes | `crates/ironclaw_host_runtime/tests/host_runtime_contract.rs` |
|
||||
| Host runtime production composition | `crates/ironclaw_host_runtime/tests/host_runtime_services_contract.rs` |
|
||||
| Dedicated Reborn E2E spine | `crates/ironclaw_host_runtime/tests/reborn_e2e_gate.rs` |
|
||||
| Capability host invoke/resume/spawn | `crates/ironclaw_capabilities/tests/capability_host_*` |
|
||||
| Dispatcher adapter selection | `crates/ironclaw_dispatcher/tests/vertical_slice_contract.rs` |
|
||||
| WASM runtime lane | `crates/ironclaw_wasm/tests/wasm_dispatch_integration.rs` and `wasm_http_adapter_contract.rs` |
|
||||
| Script runtime lane | `crates/ironclaw_scripts/tests/script_dispatch_integration.rs` and `script_http_adapter_contract.rs` |
|
||||
| MCP runtime lane | `crates/ironclaw_mcp/tests/mcp_dispatch_integration.rs` and `mcp_adapter_contract.rs` |
|
||||
| Process lifecycle | `crates/ironclaw_processes/tests/process_dispatch_integration.rs` and process service/store contracts |
|
||||
| Network policy and host HTTP egress | `crates/ironclaw_network/tests/*` plus host-runtime HTTP egress tests |
|
||||
| Secret storage/leases | `crates/ironclaw_secrets/tests/secret_store_contract.rs` plus host-runtime staged-secret tests |
|
||||
| Events/audit replay | `crates/ironclaw_events/tests/durable_log_contract.rs` and host-runtime durable-event tests |
|
||||
| Gateway product smoke | Existing Playwright scenarios under `tests/e2e/scenarios/`, especially `test_v2_*` |
|
||||
|
||||
## Dedicated Reborn E2E goal
|
||||
|
||||
The dedicated gate should answer one question:
|
||||
|
||||
```text
|
||||
Can the Reborn architecture path still execute happy, blocked, denied, failed,
|
||||
background, network, secret, event, and product-smoke paths after a change?
|
||||
```
|
||||
|
||||
It intentionally reuses existing deterministic contract/integration tests instead of duplicating them in a second test framework.
|
||||
|
||||
## Happy path spine
|
||||
|
||||
The Reborn E2E happy path is covered by a dedicated `reborn_e2e_gate.rs` spine test plus the broader host-runtime and runtime-lane tests:
|
||||
|
||||
```text
|
||||
Extension manifests
|
||||
-> ExtensionRegistry
|
||||
-> HostRuntimeServices
|
||||
-> DefaultHostRuntime / HostRuntime facade
|
||||
-> CapabilityHost authorization and run-state lifecycle
|
||||
-> RuntimeDispatcher adapter selection
|
||||
-> WASM / Script / MCP runtime adapters
|
||||
-> resource reservation and reconciliation
|
||||
-> durable runtime events
|
||||
-> structured outcome returned to caller
|
||||
```
|
||||
|
||||
Required assertions across the suite:
|
||||
|
||||
- visible capability surfaces include expected descriptors and stable surface version;
|
||||
- health reports missing runtime backends fail-closed and configured backends ready;
|
||||
- authorized invocations reach the selected runtime adapter;
|
||||
- resource reservations are reconciled or released;
|
||||
- run-state reaches the expected terminal or blocked state;
|
||||
- durable events are replayable and metadata-only;
|
||||
- runtime output is structured JSON and redacted where obligations require it.
|
||||
|
||||
## Other paths
|
||||
|
||||
The dedicated gate includes new Reborn E2E gate tests plus existing tests for these non-happy paths:
|
||||
|
||||
### Authorization and approval
|
||||
|
||||
- denied authorization fails before dispatch;
|
||||
- approval-required invocation blocks with a persisted approval request id;
|
||||
- approved resume consumes the exact lease once;
|
||||
- changed input, wrong scope/user, expired lease, or missing stores fail before dispatch;
|
||||
- unsupported obligations fail closed.
|
||||
|
||||
### Runtime availability and adapter failures
|
||||
|
||||
- missing runtime backend reports a stable missing-runtime failure;
|
||||
- runtime lane errors map to stable failure categories;
|
||||
- dispatcher/runtime adapter boundaries remain dependency-clean.
|
||||
|
||||
### Resource, process, and cancellation
|
||||
|
||||
- reservations are released on failure and reconciled on success;
|
||||
- spawned background processes publish started/completed/failed/killed transitions;
|
||||
- cancellation reaches the process graph;
|
||||
- late completion after kill does not publish a misleading success;
|
||||
- process handoffs for the same scoped capability fail closed while active.
|
||||
|
||||
### Network and secrets
|
||||
|
||||
- runtime HTTP egress is host-mediated;
|
||||
- missing staged network policy fails before transport;
|
||||
- staged secret material is consumed once;
|
||||
- runtime-supplied manual credentials are rejected;
|
||||
- raw secrets, credential-shaped values, and private host paths are not exposed in runtime-visible output, errors, events, or audit records.
|
||||
|
||||
### Event and observability
|
||||
|
||||
- durable event cursors replay runtime events;
|
||||
- stale/gap cursor behavior is deterministic;
|
||||
- event/audit records carry correlation metadata without raw payload leaks.
|
||||
|
||||
### Product smoke
|
||||
|
||||
A small Playwright smoke scenario starts an isolated `ENGINE_V2=true` gateway with the mock LLM and verifies:
|
||||
|
||||
- authenticated web shell loads;
|
||||
- text-only chat completes and persists;
|
||||
- tool-capable prompt completes through the gateway history path;
|
||||
- no duplicate assistant response is emitted for a single user turn.
|
||||
|
||||
This smoke test does not replace the full browser E2E workflow. It proves the branch remains product-bootable while the Rust Reborn gate proves architecture behavior.
|
||||
|
||||
## Local commands
|
||||
|
||||
Run the deterministic Rust Reborn gate:
|
||||
|
||||
```bash
|
||||
# Full gate
|
||||
scripts/reborn-e2e-rust.sh
|
||||
|
||||
# Or run one CI matrix group at a time
|
||||
scripts/reborn-e2e-rust.sh architecture
|
||||
scripts/reborn-e2e-rust.sh runtimes
|
||||
scripts/reborn-e2e-rust.sh substrates
|
||||
```
|
||||
|
||||
The script expands to the dedicated `reborn_e2e_gate.rs` tests plus the current Reborn boundary, host-runtime, capability-host, dispatcher, WASM, Script, MCP, process, event, filesystem, network, secret, resource, run-state, approval, and authorization contract tests. Use the script as the source of truth for local/CI parity rather than copying individual `cargo test` commands.
|
||||
|
||||
Run the gateway smoke test:
|
||||
|
||||
```bash
|
||||
cargo build --no-default-features --features libsql
|
||||
cd tests/e2e
|
||||
pip install -e .
|
||||
playwright install --with-deps chromium # on Linux CI; local macOS can omit --with-deps
|
||||
pytest scenarios/test_reborn_gateway_smoke.py -v --timeout=120
|
||||
```
|
||||
|
||||
## CI ownership
|
||||
|
||||
`reborn-e2e.yml` is intentionally separate from `e2e.yml`:
|
||||
|
||||
- Reborn changes can run a focused architecture gate without destabilizing the main browser E2E matrix.
|
||||
- The workflow is advisory by default and intentionally does not run on `merge_group`; add a merge-queue trigger only after the gate proves stable and is deliberately promoted to branch protection.
|
||||
- The Rust jobs should stay deterministic and avoid live providers.
|
||||
- The gateway job should remain a smoke test, not a second full browser matrix.
|
||||
104
scripts/reborn-e2e-rust.sh
Executable file
104
scripts/reborn-e2e-rust.sh
Executable file
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Run the deterministic Rust-side Reborn E2E gate.
|
||||
# Usage:
|
||||
# scripts/reborn-e2e-rust.sh # all groups
|
||||
# scripts/reborn-e2e-rust.sh architecture # boundary + host runtime spine
|
||||
# scripts/reborn-e2e-rust.sh runtimes # dispatcher/runtime/process lanes
|
||||
# scripts/reborn-e2e-rust.sh substrates # event/network/secret substrates
|
||||
#
|
||||
# Extra cargo test args can be passed through CARGO_TEST_ARGS, for example:
|
||||
# CARGO_TEST_ARGS='-- --nocapture' scripts/reborn-e2e-rust.sh architecture
|
||||
|
||||
group="${1:-all}"
|
||||
extra_args=${CARGO_TEST_ARGS:-"-- --nocapture"}
|
||||
|
||||
run_test() {
|
||||
local package="$1"
|
||||
local test_name="$2"
|
||||
echo "::group::cargo test -p ${package} --test ${test_name}"
|
||||
# shellcheck disable=SC2086 # extra_args intentionally expands into cargo's trailing args.
|
||||
cargo test -p "${package}" --test "${test_name}" ${extra_args}
|
||||
echo "::endgroup::"
|
||||
}
|
||||
|
||||
run_architecture() {
|
||||
run_test ironclaw_architecture reborn_dependency_boundaries
|
||||
run_test ironclaw_host_runtime host_runtime_contract
|
||||
run_test ironclaw_host_runtime host_runtime_services_contract
|
||||
run_test ironclaw_host_runtime reborn_e2e_gate
|
||||
run_test ironclaw_host_runtime reborn_invoke_vertical_slice
|
||||
run_test ironclaw_host_runtime runtime_http_egress_contract
|
||||
run_test ironclaw_host_runtime builtin_obligation_handler_contract
|
||||
run_test ironclaw_host_runtime obligation_services_composition_contract
|
||||
run_test ironclaw_host_runtime production_trust_contract
|
||||
run_test ironclaw_capabilities capability_boundary_contract
|
||||
run_test ironclaw_capabilities capability_host_contract
|
||||
run_test ironclaw_capabilities capability_host_dispatcher_integration
|
||||
run_test ironclaw_capabilities capability_host_process_integration
|
||||
run_test ironclaw_capabilities capability_host_run_state_contract
|
||||
run_test ironclaw_capabilities capability_host_spawn_contract
|
||||
run_test ironclaw_capabilities capability_obligation_handler_contract
|
||||
}
|
||||
|
||||
run_runtimes() {
|
||||
run_test ironclaw_dispatcher boundary_contract
|
||||
run_test ironclaw_dispatcher dispatch_contract
|
||||
run_test ironclaw_dispatcher event_dispatch_contract
|
||||
run_test ironclaw_dispatcher runtime_dispatcher_integration
|
||||
run_test ironclaw_dispatcher vertical_slice_contract
|
||||
run_test ironclaw_wasm wasm_dispatch_integration
|
||||
run_test ironclaw_wasm wasm_http_adapter_contract
|
||||
run_test ironclaw_wasm wit_tool_runtime_contract
|
||||
run_test ironclaw_scripts script_dispatch_integration
|
||||
run_test ironclaw_scripts script_http_adapter_contract
|
||||
run_test ironclaw_scripts script_runner_contract
|
||||
run_test ironclaw_mcp mcp_adapter_contract
|
||||
run_test ironclaw_mcp mcp_dispatch_integration
|
||||
run_test ironclaw_processes process_dispatch_integration
|
||||
run_test ironclaw_processes process_host_contract
|
||||
run_test ironclaw_processes process_services_contract
|
||||
run_test ironclaw_processes process_store_contract
|
||||
}
|
||||
|
||||
run_substrates() {
|
||||
run_test ironclaw_events durable_log_contract
|
||||
run_test ironclaw_filesystem catalog_contract
|
||||
run_test ironclaw_filesystem filesystem_contract
|
||||
run_test ironclaw_network boundary_contract
|
||||
run_test ironclaw_network network_http_egress_contract
|
||||
run_test ironclaw_network network_policy_contract
|
||||
run_test ironclaw_secrets boundary_contract
|
||||
run_test ironclaw_secrets secret_store_contract
|
||||
run_test ironclaw_resources resource_governor_contract
|
||||
run_test ironclaw_run_state approval_resolution_contract
|
||||
run_test ironclaw_run_state run_state_contract
|
||||
run_test ironclaw_approvals approval_resolution_contract
|
||||
run_test ironclaw_approvals boundary_contract
|
||||
run_test ironclaw_authorization boundary_contract
|
||||
run_test ironclaw_authorization capability_access_contract
|
||||
run_test ironclaw_authorization capability_lease_contract
|
||||
}
|
||||
|
||||
case "${group}" in
|
||||
architecture)
|
||||
run_architecture
|
||||
;;
|
||||
runtimes)
|
||||
run_runtimes
|
||||
;;
|
||||
substrates)
|
||||
run_substrates
|
||||
;;
|
||||
all)
|
||||
run_architecture
|
||||
run_runtimes
|
||||
run_substrates
|
||||
;;
|
||||
*)
|
||||
echo "unknown Reborn E2E group: ${group}" >&2
|
||||
echo "expected one of: architecture, runtimes, substrates, all" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
265
tests/e2e/scenarios/test_reborn_gateway_smoke.py
Normal file
265
tests/e2e/scenarios/test_reborn_gateway_smoke.py
Normal file
@@ -0,0 +1,265 @@
|
||||
"""Dedicated Reborn gateway smoke E2E.
|
||||
|
||||
This is intentionally small. The Rust Reborn gate proves the host/runtime
|
||||
architecture. This Playwright/API smoke test proves the reborn-main branch still
|
||||
boots an isolated ENGINE_V2 gateway, serves the browser shell, persists a normal
|
||||
chat turn, and completes a simple tool-capable turn without duplicate terminal
|
||||
assistant responses.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
from playwright.async_api import expect
|
||||
|
||||
from helpers import AUTH_TOKEN, SEL, api_get, api_post, wait_for_ready
|
||||
|
||||
_REBORN_GATEWAY_DB_TMPDIR = tempfile.TemporaryDirectory(prefix="ironclaw-reborn-gateway-e2e-")
|
||||
_REBORN_GATEWAY_HOME_TMPDIR = tempfile.TemporaryDirectory(prefix="ironclaw-reborn-gateway-e2e-home-")
|
||||
|
||||
|
||||
def _reserve_loopback_sockets(count: int) -> list[socket.socket]:
|
||||
sockets: list[socket.socket] = []
|
||||
try:
|
||||
while len(sockets) < count:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
sockets.append(sock)
|
||||
return sockets
|
||||
except Exception:
|
||||
for sock in sockets:
|
||||
sock.close()
|
||||
raise
|
||||
|
||||
|
||||
def _close_reserved_sockets(sockets: list[socket.socket]) -> None:
|
||||
for sock in sockets:
|
||||
if sock.fileno() != -1:
|
||||
sock.close()
|
||||
|
||||
|
||||
def _forward_coverage_env(env: dict[str, str]) -> None:
|
||||
for key, value in os.environ.items():
|
||||
if key.startswith(("CARGO_LLVM_COV", "LLVM_")) or key in {
|
||||
"CARGO_ENCODED_RUSTFLAGS",
|
||||
"CARGO_INCREMENTAL",
|
||||
}:
|
||||
env[key] = value
|
||||
|
||||
|
||||
async def _stop_process(proc, *, sig=signal.SIGINT, timeout: float = 10) -> None:
|
||||
async def _drain_pipes() -> None:
|
||||
try:
|
||||
await asyncio.wait_for(proc.communicate(), timeout=1)
|
||||
except (asyncio.TimeoutError, ValueError):
|
||||
pass
|
||||
|
||||
if proc.returncode is not None:
|
||||
await _drain_pipes()
|
||||
return
|
||||
|
||||
try:
|
||||
proc.send_signal(sig)
|
||||
except ProcessLookupError:
|
||||
await _drain_pipes()
|
||||
return
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=timeout)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
await _drain_pipes()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
async def reborn_gateway_server(ironclaw_binary, mock_llm_server):
|
||||
"""Start an isolated gateway configured for the Reborn/V2 product shell."""
|
||||
home_dir = _REBORN_GATEWAY_HOME_TMPDIR.name
|
||||
base_dir = os.path.join(home_dir, ".ironclaw")
|
||||
os.makedirs(base_dir, exist_ok=True)
|
||||
reserved_sockets = _reserve_loopback_sockets(2)
|
||||
gateway_port = reserved_sockets[0].getsockname()[1]
|
||||
http_port = reserved_sockets[1].getsockname()[1]
|
||||
|
||||
env = {
|
||||
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
||||
"HOME": home_dir,
|
||||
"IRONCLAW_BASE_DIR": base_dir,
|
||||
"RUST_LOG": "ironclaw=info",
|
||||
"RUST_BACKTRACE": "1",
|
||||
"ENGINE_V2": "true",
|
||||
"AGENT_AUTO_APPROVE_TOOLS": "true",
|
||||
"GATEWAY_ENABLED": "true",
|
||||
"GATEWAY_HOST": "127.0.0.1",
|
||||
"GATEWAY_PORT": str(gateway_port),
|
||||
"GATEWAY_AUTH_TOKEN": AUTH_TOKEN,
|
||||
"GATEWAY_USER_ID": "reborn-gateway-e2e-user",
|
||||
"HTTP_HOST": "127.0.0.1",
|
||||
"HTTP_PORT": str(http_port),
|
||||
"CLI_ENABLED": "false",
|
||||
"LLM_BACKEND": "openai_compatible",
|
||||
"LLM_BASE_URL": mock_llm_server,
|
||||
"LLM_API_KEY": "mock-api-key",
|
||||
"LLM_MODEL": "mock-model",
|
||||
"DATABASE_BACKEND": "libsql",
|
||||
"LIBSQL_PATH": os.path.join(_REBORN_GATEWAY_DB_TMPDIR.name, "reborn-gateway-e2e.db"),
|
||||
"SANDBOX_ENABLED": "false",
|
||||
"SKILLS_ENABLED": "false",
|
||||
"ROUTINES_ENABLED": "false",
|
||||
"HEARTBEAT_ENABLED": "false",
|
||||
"EMBEDDING_ENABLED": "false",
|
||||
"WASM_ENABLED": "false",
|
||||
"ONBOARD_COMPLETED": "true",
|
||||
}
|
||||
_forward_coverage_env(env)
|
||||
|
||||
_close_reserved_sockets(reserved_sockets)
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
ironclaw_binary,
|
||||
"--no-onboard",
|
||||
stdin=asyncio.subprocess.DEVNULL,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
base_url = f"http://127.0.0.1:{gateway_port}"
|
||||
|
||||
try:
|
||||
await wait_for_ready(f"{base_url}/api/health", timeout=60)
|
||||
yield base_url
|
||||
except TimeoutError:
|
||||
if proc.returncode is None:
|
||||
await _stop_process(proc, timeout=2)
|
||||
stderr_text = ""
|
||||
if proc.stderr:
|
||||
try:
|
||||
stderr_bytes = await asyncio.wait_for(proc.stderr.read(8192), timeout=2)
|
||||
stderr_text = stderr_bytes.decode("utf-8", errors="replace")
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
pytest.fail(
|
||||
f"Reborn gateway smoke server failed to start on port {gateway_port}.\n"
|
||||
f"stderr:\n{stderr_text}"
|
||||
)
|
||||
finally:
|
||||
if proc.returncode is None:
|
||||
await _stop_process(proc, sig=signal.SIGINT, timeout=10)
|
||||
if proc.returncode is None:
|
||||
await _stop_process(proc, sig=signal.SIGTERM, timeout=5)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def reborn_gateway_page(reborn_gateway_server, browser):
|
||||
context = await browser.new_context(viewport={"width": 1280, "height": 720})
|
||||
page = await context.new_page()
|
||||
await page.goto(f"{reborn_gateway_server}/?token={AUTH_TOKEN}")
|
||||
await page.wait_for_selector(SEL["auth_screen"], state="hidden", timeout=15000)
|
||||
await page.wait_for_function(
|
||||
"() => typeof sseHasConnectedBefore !== 'undefined' && sseHasConnectedBefore === true",
|
||||
timeout=10000,
|
||||
)
|
||||
yield page
|
||||
await context.close()
|
||||
|
||||
|
||||
async def _create_thread(base_url: str) -> str:
|
||||
response = await api_post(base_url, "/api/chat/thread/new", timeout=15)
|
||||
response.raise_for_status()
|
||||
return response.json()["id"]
|
||||
|
||||
|
||||
async def _send_message(base_url: str, thread_id: str, content: str) -> None:
|
||||
response = await api_post(
|
||||
base_url,
|
||||
"/api/chat/send",
|
||||
json={"content": content, "thread_id": thread_id},
|
||||
timeout=30,
|
||||
)
|
||||
assert response.status_code in (200, 202), response.text
|
||||
|
||||
|
||||
async def _wait_for_terminal_turn(
|
||||
base_url: str,
|
||||
thread_id: str,
|
||||
expected_user_input: str,
|
||||
*,
|
||||
timeout: float = 45.0,
|
||||
) -> dict:
|
||||
last_history = {}
|
||||
for _ in range(int(timeout * 2)):
|
||||
response = await api_get(
|
||||
base_url,
|
||||
f"/api/chat/history?thread_id={thread_id}",
|
||||
timeout=15,
|
||||
)
|
||||
response.raise_for_status()
|
||||
history = response.json()
|
||||
last_history = history
|
||||
turns = history.get("turns", [])
|
||||
matching_turns = [
|
||||
turn
|
||||
for turn in turns
|
||||
if expected_user_input in (turn.get("user_input") or "")
|
||||
]
|
||||
if matching_turns and (matching_turns[-1].get("response") or "").strip():
|
||||
return matching_turns[-1]
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
raise AssertionError(
|
||||
f"Timed out waiting for terminal turn containing {expected_user_input!r}. "
|
||||
f"Last history: {last_history}"
|
||||
)
|
||||
|
||||
|
||||
async def test_reborn_gateway_loads_engine_v2_shell(reborn_gateway_page):
|
||||
"""The isolated Reborn smoke gateway should boot the ENGINE_V2 shell."""
|
||||
chat_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="chat"))
|
||||
missions_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="missions"))
|
||||
routines_tab = reborn_gateway_page.locator(SEL["tab_button"].format(tab="routines"))
|
||||
|
||||
await expect(chat_tab).to_be_visible()
|
||||
await expect(missions_tab).to_be_visible()
|
||||
await expect(routines_tab).to_be_hidden()
|
||||
|
||||
|
||||
async def test_reborn_gateway_persists_text_and_tool_turns_without_duplicate_response(
|
||||
reborn_gateway_server,
|
||||
):
|
||||
"""A text turn and an auto-approved tool turn should each produce one terminal response."""
|
||||
thread_id = await _create_thread(reborn_gateway_server)
|
||||
|
||||
text_prompt = "reborn gateway smoke: what is 2+2?"
|
||||
await _send_message(reborn_gateway_server, thread_id, text_prompt)
|
||||
text_turn = await _wait_for_terminal_turn(reborn_gateway_server, thread_id, text_prompt)
|
||||
assert "4" in text_turn.get("response", "")
|
||||
|
||||
tool_prompt = "echo reborn gateway smoke tool result"
|
||||
await _send_message(reborn_gateway_server, thread_id, tool_prompt)
|
||||
tool_turn = await _wait_for_terminal_turn(reborn_gateway_server, thread_id, tool_prompt)
|
||||
assert "reborn gateway smoke tool result" in tool_turn.get("response", "").lower()
|
||||
|
||||
tool_calls = tool_turn.get("tool_calls", [])
|
||||
assert tool_calls, f"Expected persisted tool call metadata, got: {tool_turn}"
|
||||
assert any(call.get("name") == "echo" and call.get("has_result") for call in tool_calls)
|
||||
|
||||
history_response = await api_get(
|
||||
reborn_gateway_server,
|
||||
f"/api/chat/history?thread_id={thread_id}",
|
||||
timeout=15,
|
||||
)
|
||||
history_response.raise_for_status()
|
||||
matching_tool_turns = [
|
||||
turn
|
||||
for turn in history_response.json().get("turns", [])
|
||||
if tool_prompt in (turn.get("user_input") or "")
|
||||
and (turn.get("response") or "").strip()
|
||||
]
|
||||
assert len(matching_tool_turns) == 1, (
|
||||
"Expected one terminal assistant response for the tool prompt, got "
|
||||
f"{len(matching_tool_turns)} turns: {matching_tool_turns}"
|
||||
)
|
||||
Reference in New Issue
Block a user