diff --git a/crates/loop/ironclaw_agent_loop/src/executor/tests/compaction.rs b/crates/loop/ironclaw_agent_loop/src/executor/tests/compaction.rs index b5a300f2e2..f6f8705431 100644 --- a/crates/loop/ironclaw_agent_loop/src/executor/tests/compaction.rs +++ b/crates/loop/ironclaw_agent_loop/src/executor/tests/compaction.rs @@ -846,6 +846,31 @@ async fn model_context_overflow_retries_through_canonical_compaction_stage() { "one failed model attempt must produce exactly one recovery numerator event" ); + let recovery_checkpoint = host + .staged_payloads() + .into_iter() + .filter(|request| request.kind == LoopCheckpointKind::BeforeModel) + .filter_map(|request| { + LoopExecutionState::from_checkpoint_payload( + &request.payload, + CheckpointKind::BeforeModel, + ) + .ok() + }) + .find(|state| { + state + .recovery_state + .attempts_for(crate::state::RecoveryAttemptClass::ModelContextOverflow) + == 1 + && state.compaction_state.force_compact_on_next_iteration + }) + .expect("recovery checkpoint persists the consumed attempt and compaction request"); + assert!( + recovery_checkpoint + .pending_model_error_observation + .is_none() + ); + let final_state = final_staged_state(&host); assert_eq!( final_state.compaction_state.last_compacted_through_seq, @@ -854,6 +879,55 @@ async fn model_context_overflow_retries_through_canonical_compaction_stage() { assert!(!final_state.compaction_state.force_compact_on_next_iteration); } +#[tokio::test] +async fn second_model_context_overflow_aborts_without_another_compaction() { + let overflow = || { + AgentLoopHostError::new( + AgentLoopHostErrorKind::ContextOverflow, + "model request exceeded its context budget", + ) + }; + let host = MockHost::new(Vec::new()) + .with_model_errors(vec![overflow(), overflow()]) + .with_prompt_compaction_indexes(vec![ + vec![compaction_metadata(1, LoopContextCompactionKind::User, 10)], + active_task_preserving_compaction_index(), + Vec::new(), + ]) + .with_compaction_result(Ok(LoopCompactionResponse { + summary_artifact_id: LoopSummaryArtifactId::new("summary:overflow-once") + .expect("valid summary id"), + compression_ratio_ppm: 100_000, + redacted_leak_count: 0, + })); + let executor = CanonicalAgentLoopExecutor; + let state = LoopExecutionState::initial_for_run(host.run_context()); + + let exit = executor + .execute_family(&crate::families::default(), &host, state) + .await + .expect("execute"); + + assert!(matches!(exit, LoopExit::Failed(_))); + assert_eq!(host.model_requests().len(), 2); + assert_eq!( + host.progress_events() + .into_iter() + .filter(|event| matches!(event, LoopProgressEvent::CompactionStarted { .. })) + .count(), + 1, + "a second overflow must not start another compaction" + ); + let final_state = final_staged_state(&host); + assert_eq!( + final_state + .recovery_state + .attempts_for(crate::state::RecoveryAttemptClass::ModelContextOverflow), + 2 + ); + assert!(final_state.pending_model_error_observation.is_none()); +} + /// D-A integration: the `force_compact_initiator` threaded through state by /// PostCapabilityStage must survive the iteration boundary and appear in the /// `CompactionStarted` event emitted by `PromptCompactionStep` on iteration 2. diff --git a/crates/loop/ironclaw_agent_loop/src/executor/tests/prompt_stage.rs b/crates/loop/ironclaw_agent_loop/src/executor/tests/prompt_stage.rs index ce3118573b..6140903fcf 100644 --- a/crates/loop/ironclaw_agent_loop/src/executor/tests/prompt_stage.rs +++ b/crates/loop/ironclaw_agent_loop/src/executor/tests/prompt_stage.rs @@ -305,34 +305,32 @@ async fn prompt_stage_cancellation_after_prompt_bundle_returns_cancelled_exit() } #[tokio::test] -async fn model_context_overflow_exhaustion_gives_model_one_observation_assisted_attempt() { +async fn model_context_overflow_never_receives_an_observation_retry() { let overflow = || { AgentLoopHostError::new( AgentLoopHostErrorKind::ContextOverflow, "model request exceeded its context budget", ) }; - let host = MockHost::new(vec![reply_response()]).with_model_errors(vec![ - overflow(), - overflow(), - overflow(), - ]); + let host = MockHost::new(Vec::new()).with_model_errors(vec![overflow(), overflow()]); let executor = CanonicalAgentLoopExecutor; let state = LoopExecutionState::initial_for_run(host.run_context()); let exit = executor .execute_family(&crate::families::default(), &host, state) .await - .expect("context-overflow observation should let the model recover"); + .expect("second context overflow returns a terminal exit"); - assert!(matches!(exit, LoopExit::Completed(_))); + assert!(matches!(exit, LoopExit::Failed(_))); let requests = host.model_requests(); - assert_eq!(requests.len(), 4); - assert!(requests[3].inline_messages.iter().any(|message| { - message - .safe_body - .as_str() - .contains("context overflowed; use the available context and continue") + assert_eq!(requests.len(), 2); + assert!(requests.iter().all(|request| { + request.inline_messages.iter().all(|message| { + !message + .safe_body + .as_str() + .contains("context overflowed; use the available context and continue") + }) })); } diff --git a/crates/loop/ironclaw_agent_loop/src/state/model_recovery.rs b/crates/loop/ironclaw_agent_loop/src/state/model_recovery.rs index a330e69850..298c09b49a 100644 --- a/crates/loop/ironclaw_agent_loop/src/state/model_recovery.rs +++ b/crates/loop/ironclaw_agent_loop/src/state/model_recovery.rs @@ -17,13 +17,6 @@ pub struct ModelErrorRecoveryObservation { } impl ModelErrorRecoveryObservation { - pub fn context_overflow() -> Self { - Self { - schema_version: MODEL_ERROR_OBSERVATION_SCHEMA_VERSION, - detail: ModelErrorRecoveryDetail::ContextOverflow, - } - } - pub fn content_filtered() -> Self { Self { schema_version: MODEL_ERROR_OBSERVATION_SCHEMA_VERSION, diff --git a/crates/loop/ironclaw_agent_loop/src/strategies/recovery.rs b/crates/loop/ironclaw_agent_loop/src/strategies/recovery.rs index 2e4bcdc5d7..cffdd58df9 100644 --- a/crates/loop/ironclaw_agent_loop/src/strategies/recovery.rs +++ b/crates/loop/ironclaw_agent_loop/src/strategies/recovery.rs @@ -247,8 +247,8 @@ pub(crate) enum RetryScope { /// attempt before aborting. Provider outages (5xx storms) routinely outlast /// a couple of quick retries; a long-running agentic turn must ride them out /// rather than discard all prior work. -/// - Retries `ContextOverflow` at iteration scope with `ShrinkContext`, then -/// gives the compacted prompt one observation-assisted attempt before aborting. +/// - Retries `ContextOverflow` exactly once at iteration scope with +/// `ShrinkContext`; the checkpointed attempt bit makes a second overflow terminal. /// - Retries `StaleRequest` at iteration scope (rebuilding the capability /// surface and prompt bundle) up to [`Self::max_attempts_per_class`] times, /// then gives the refreshed iteration one typed observation-assisted @@ -374,13 +374,7 @@ impl RecoveryStrategy for DefaultRecoveryStrategy { ModelErrorRecoveryObservation::stale_request(), ) } - ModelErrorClass::ContextOverflow => retry_observe_or_abort( - state, - self.max_attempts_per_class, - RetryScope::Iteration, - |_| Some(RetryAlteration::ShrinkContext), - ModelErrorRecoveryObservation::context_overflow(), - ), + ModelErrorClass::ContextOverflow => recover_context_overflow_once(state), ModelErrorClass::InvalidOutput => { let reason = ModelInvalidOutputDetailReason::from_safe_summary(err.safe_summary.as_str()); @@ -493,6 +487,26 @@ fn retry_or_abort( } } +fn recover_context_overflow_once(state: &LoopExecutionState) -> RecoveryOutcome { + let attempt_class = RecoveryAttemptClass::ModelContextOverflow; + let attempts = state.recovery_state.attempts_for(attempt_class); + let next = state + .recovery_state + .with_incremented_attempts_for(attempt_class); + if attempts == 0 { + RecoveryOutcome::Retry { + recovery: next, + scope: RetryScope::Iteration, + alter: Some(RetryAlteration::ShrinkContext), + } + } else { + RecoveryOutcome::Abort { + recovery: next, + failure_kind: LoopFailureKind::ModelError, + } + } +} + fn retry_observe_or_abort( state: &LoopExecutionState, max_attempts_per_class: u32, @@ -1426,7 +1440,7 @@ mod tests { } #[tokio::test] - async fn model_context_overflow_retries_then_observes_once_before_abort() { + async fn model_context_overflow_compacts_once_then_aborts() { let strategy = DefaultRecoveryStrategy::default(); let state = state_with_no_attempts(); @@ -1434,7 +1448,7 @@ mod tests { .on_model_error(&state, &model_err(ModelErrorClass::ContextOverflow)) .await; - match outcome { + let recovery = match outcome { RecoveryOutcome::Retry { recovery, scope, @@ -1446,43 +1460,33 @@ mod tests { ); assert_eq!(scope, RetryScope::Iteration); assert_eq!(alter, Some(RetryAlteration::ShrinkContext)); - } - other => panic!("expected context overflow retry, got {other:?}"), - } - - let state = state_with_attempts_for(2, RecoveryAttemptClass::ModelContextOverflow); - let outcome = strategy - .on_model_error(&state, &model_err(ModelErrorClass::ContextOverflow)) - .await; - let recovery = match outcome { - RecoveryOutcome::ModelErrorObservation { - recovery, - scope, - alter, - observation, - } => { - assert_eq!(scope, RetryScope::Iteration); - assert_eq!(alter, Some(RetryAlteration::ShrinkContext)); - assert_eq!( - observation, - ModelErrorRecoveryObservation::context_overflow() - ); recovery } - other => panic!("expected context-overflow observation, got {other:?}"), + other => panic!("expected context overflow retry, got {other:?}"), }; - let mut state = state_with_no_attempts(); - state.recovery_state = recovery; + + let mut resumed = state_with_no_attempts(); + resumed.recovery_state = recovery; let outcome = strategy - .on_model_error(&state, &model_err(ModelErrorClass::ContextOverflow)) + .on_model_error(&resumed, &model_err(ModelErrorClass::ContextOverflow)) .await; - assert!(matches!( - outcome, + match outcome { RecoveryOutcome::Abort { - failure_kind: LoopFailureKind::ModelError, - .. + recovery, + failure_kind, + } => { + assert_eq!(failure_kind, LoopFailureKind::ModelError); + assert_eq!( + recovery.attempts_for(RecoveryAttemptClass::ModelContextOverflow), + 2 + ); + assert!( + !recovery + .observation_attempted_for(ModelErrorObservationClass::ContextOverflow) + ); } - )); + other => panic!("second context overflow must abort, got {other:?}"), + } } #[tokio::test] diff --git a/docs/internal/reborn/contracts/turns-agent-loop.md b/docs/internal/reborn/contracts/turns-agent-loop.md index 0d336823aa..e4b16fb106 100644 --- a/docs/internal/reborn/contracts/turns-agent-loop.md +++ b/docs/internal/reborn/contracts/turns-agent-loop.md @@ -138,6 +138,13 @@ Rules: appended only when another model call is safe and possible, while a user-visible terminal explanation is host-authored from a typed failure category and must not imply that the failed model saw it; +- context overflow consumes one checkpointed, iteration-scoped recovery: + the loop requests canonical compaction, rebuilds the prompt from the durable + barrier, and resumes the interrupted model stage; the recovery attempt and + typed recovery event are durable, but provider diagnostics and synthetic + error text never enter model context; +- a second context overflow after that checkpoint is terminal. It must not + trigger another compaction or receive a model-error observation attempt; - model provider authentication or credential failures do not blindly retry the same rejected route; when no already-authorized fallback route exists, the run fails with a durable credential/account remediation; diff --git a/docs/internal/research/pi-agent-deep-dive.md b/docs/internal/research/pi-agent-deep-dive.md index 39967d1e88..152eb00caf 100644 --- a/docs/internal/research/pi-agent-deep-dive.md +++ b/docs/internal/research/pi-agent-deep-dive.md @@ -420,7 +420,7 @@ refs below into the IronClaw tree, verified 2026-08-01). | Step cap | None | 1,024-iteration backstop with a model-visible terminal warning first (`strategies/budget.rs:35`) | | Termination | Text-only response + empty queues | Rich stop heuristics: reply-only turn, no-progress ×3, repeated-call signature, diminishing returns, rejected replies (`strategies/stop.rs:307-385`) | | LLM retry | Outside the loop; error message popped from LLM context, kept in history | In-stage, per-error-class budgets up to 45 attempts (`strategies/recovery.rs:58,333-450`); error surfaces as an *ephemeral* inline message, never a durable message | -| Context overflow | Compact + retry exactly once | ShrinkContext retry ×2 + one observation-assisted attempt, forced compaction (`recovery.rs:376-382`) | +| Context overflow | Compact + retry exactly once | Checkpoint the consumed recovery, force canonical compaction, rebuild the prompt, and retry exactly once; a second overflow fails without a model observation (`strategies/recovery.rs`) | | Tool execution | Parallel with sequential preflight; source-order persistence; per-file mutation queue | **Sequential always** — `Parallel` verdict only controls park behavior; port iterates one at a time (`ironclaw_loop_host/src/capability_port.rs:1886-1910`) | | `length`-stopped tool calls | Never executed; per-call synthesized error results tell the model to re-issue | Never executed; whole response becomes `OutputTruncated`, one observation-assisted continuation then abort (`model_gateway.rs:1683-1688,1816`) | | Steering | Three queues with explicit drain points | Steering + follow-up drains at two points (`canonical.rs:77-99,277-314`); `allow_steering`/`allow_interrupt` policy flags exist but are never consulted (`strategies/drain.rs:33-41`) | diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 8c6f1f6d19..8d43d1e311 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -233,6 +233,7 @@ One thread, whole real turn. Grouped by what the user experiences. | Typing again while the assistant is working queues the message and it gets picked up mid-run | `steering.rs` | | A flaky model provider is retried and recovered from, with typed errors | `model_recovery.rs` | | A cumulative compaction barrier supersedes earlier summaries and raw covered history in the model prompt while every original row remains durable | `model_recovery.rs::cumulative_compaction_barrier_replaces_earlier_summaries_and_raw_history` | +| Context overflow checkpoints one compact-and-resume attempt; a second overflow fails without another compaction or model observation | `model_recovery.rs::context_overflow_compacts_once_and_resumes`, `model_recovery.rs::second_context_overflow_fails_after_one_compaction` | | A turn receives the expected tool results after each model iteration | `golden_payload.rs` | | A turn that reads two file ranges in parallel receives both results in the requested order | `golden_payload.rs` | | Approaching the run limit surfaces a recoverable warning, while repeated capability calls receive one advisory warning and may continue | `terminal_warning.rs` | diff --git a/tests/integration/model_recovery.rs b/tests/integration/model_recovery.rs index d116464a5c..e239fd68e2 100644 --- a/tests/integration/model_recovery.rs +++ b/tests/integration/model_recovery.rs @@ -269,7 +269,7 @@ async fn cumulative_compaction_barrier_replaces_earlier_summaries_and_raw_histor } #[tokio::test] -async fn context_overflow_recovers_with_model_visible_observation() { +async fn context_overflow_compacts_once_and_resumes() { // Seed one oversized user message so forced compaction exercises the real // compactor instead of taking its safe "nothing eligible" skip path. let input_secret = concat!("AKIA", "IOSFODNN7EXAMPLE"); @@ -282,7 +282,8 @@ async fn context_overflow_recovers_with_model_visible_observation() { "compacted recovery history\n-----BEGIN ENCRYPTED PRIVATE KEY-----\n{output_secret}\n-----END ENCRYPTED PRIVATE KEY-----\nretained" ); let harness = RebornIntegrationHarness::test_default() - .context_overflow_model_after(3, 3) + .with_durable_milestone_event_store_for_test() + .context_overflow_model_after(3, 1) .script([ RebornScriptedReply::text("first setup reply"), RebornScriptedReply::text("second setup reply"), @@ -312,17 +313,17 @@ async fn context_overflow_recovers_with_model_visible_observation() { harness .submit_turn("answer after compacting") .await - .expect("turn recovers after context overflow exhausts normal retries"); + .expect("turn recovers after one context overflow"); harness .assert_reply_contains("recovered after context overflow") .await .expect("recovered reply persisted"); harness - .assert_model_message_content_contains( + .assert_model_message_content_not_contains( "model error observation: context overflowed; use the available context and continue", ) .await - .expect("recovery request carries the typed context-overflow observation"); + .expect("overflow recovery resumes from compaction without an observation retry"); harness .assert_model_message_content_contains("compacted recovery history") .await @@ -372,21 +373,90 @@ async fn context_overflow_recovers_with_model_visible_observation() { .await .expect("the durable compaction summary contains only redaction markers"); harness - .assert_interactive_model_provider_call_count(7) + .assert_interactive_model_provider_call_count(5) .await - .expect("setup and context-overflow recovery use the bounded interactive budget"); + .expect("setup, one overflow, and one resumed request use the bounded budget"); harness .assert_text_model_provider_call_count_at_least(1) .await .expect("context overflow performs a real text-only compaction inference"); harness - .assert_model_message_content_occurrences("model error observation", 1) + .assert_model_message_content_occurrences("model error observation", 0) .await - .expect("context overflow injects exactly one recovery observation"); + .expect("context overflow does not inject an extra observation attempt"); harness .assert_model_message_content_not_contains(&CONTEXT_OVERFLOW_USED_TOKENS.to_string()) .await .expect("provider diagnostics do not enter the recovery prompt"); + harness + .assert_model_recovery_class( + LoopRecoveryClass::ModelContextOverflow, + LoopRecoveryClass::ModelInvalidOutput, + ) + .await + .expect("the consumed overflow recovery is recorded durably"); +} + +#[tokio::test] +async fn second_context_overflow_fails_after_one_compaction() { + let harness = RebornIntegrationHarness::test_default() + .with_turn_event_sink() + .with_durable_milestone_event_store_for_test() + .context_overflow_model_after(3, 2) + .script([ + RebornScriptedReply::text("first setup reply"), + RebornScriptedReply::text("second setup reply"), + RebornScriptedReply::text("third setup reply"), + RebornScriptedReply::text("single overflow compaction summary"), + ]) + .build() + .await + .expect("harness builds"); + harness + .submit_turn("first setup turn") + .await + .expect("first setup turn"); + harness + .submit_turn("second setup turn") + .await + .expect("second setup turn"); + harness + .submit_turn(&format!("third setup turn {}", "history ".repeat(5_000))) + .await + .expect("third setup turn"); + + let run_id = harness + .submit_turn_async("fail after one overflow recovery") + .await + .expect("turn submitted"); + harness + .wait_for_status(run_id, TurnStatus::Failed) + .await + .expect("second overflow fails the run"); + + harness + .assert_summary_artifact_count_at_least(1) + .await + .expect("exactly one recovery path reaches durable compaction"); + harness + .assert_interactive_model_provider_call_count(5) + .await + .expect("setup plus two overflow attempts are bounded"); + harness + .assert_model_message_content_occurrences("model error observation", 0) + .await + .expect("a second overflow does not receive an observation retry"); + harness + .assert_turn_event_recorded(TurnEventKind::Failed) + .await + .expect("terminal overflow failure is durable"); + harness + .assert_model_recovery_class( + LoopRecoveryClass::ModelContextOverflow, + LoopRecoveryClass::ModelInvalidOutput, + ) + .await + .expect("the single consumed recovery remains durably recorded"); } #[tokio::test]