mirror of
https://github.com/nearai/ironclaw.git
synced 2026-09-02 23:56:24 +08:00
feat(traces): carry confidence aggregates on contributed envelopes
The reduction existed and nothing called it. This closes the join. The correlation looked missing at first and is not. The model gateway writes the logprob sidecar keyed by the run id from HostManagedModelRequest, and the contribution path rebuilds a trace from a transcript afterwards, carrying no identity of its own - the reconstructed thread records set turn_run_id: None by construction. But TurnLifecycleEvent, which is what triggers capture, already carries run_id, so the identity exists at the one place that needs to supply it. TraceClientAutonomousCaptureRequest gains run_id, threaded in for the same reason and in the same shape as outcome_override: the contribution path cannot derive something that only existed during generation, so a caller that knows it passes it. capture_turn_trace supplies event.run_id, which is the production path. prepare_autonomous_envelope_from_messages then sets training_dynamics from whatever the sidecar holds for that run. build_autonomous_envelope_from_messages delegates to it, so both entry points are covered. None means no aggregates, which is the ordinary case rather than a degradation - capture is off by default, so almost every envelope will carry nothing here and that is correct. Raw distributions never move. They stay in the sidecar; only the four reduced numbers reach the envelope. They cannot cross the ingest boundary anyway - the limit is 2 MiB and top-5 logprobs for a typical trace is several times that - and they are more sensitive than the text they describe. 2 tests. One writes a sidecar under a known run id and asserts the envelope built for that run carries the matching mean and bucket, which is the end-to-end join; it also asserts correctness stays unset, since that needs an outcome signal rather than confidence. The other asserts a run without capture contributes nothing. Baseline 1126 passed / 0 failed; 1128 / 0 with this change, run with --no-fail-fast because error::tests::refused_http_connections_are_transient is environmentally flaky (expects a refused connection, intermittently gets a reset). clippy and fmt clean. All runs used IRONCLAW_DISABLE_OS_KEYCHAIN=1, matching every CI workflow: integration tests link the non-cfg(test) lib, so the crate's own cfg!(test) suppression does not cover them and the real keychain would prompt. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -218,6 +218,10 @@ pub(crate) async fn capture_turn_trace(
|
||||
}
|
||||
|
||||
let turn_failed = matches!(event.kind, TurnEventKind::Failed);
|
||||
// The same identity the model gateway writes the logprob sidecar under, so
|
||||
// a turn's confidence aggregates can be found again after the fact. The
|
||||
// transcript this trace is rebuilt from carries no such identity.
|
||||
let run_id = event.run_id.to_string();
|
||||
let outcome = TraceClientHost
|
||||
.prepare_autonomous_envelope_from_messages(TraceClientAutonomousCaptureRequest {
|
||||
scope: TraceClientScope::user(scope.clone()),
|
||||
@@ -231,6 +235,7 @@ pub(crate) async fn capture_turn_trace(
|
||||
// Reborn thread transcripts carry no structured outcome payload;
|
||||
// the lifecycle event's terminal status is authoritative.
|
||||
outcome_override: turn_failed.then_some(trace::TaskSuccess::Failure),
|
||||
run_id: Some(run_id.as_str()),
|
||||
})
|
||||
.await;
|
||||
match outcome {
|
||||
|
||||
@@ -46,6 +46,17 @@ pub struct TraceClientAutonomousCaptureRequest<'a> {
|
||||
/// authoritative than the response-presence fallback used when message
|
||||
/// transcripts carry no structured outcome payload.
|
||||
pub outcome_override: Option<trace::TaskSuccess>,
|
||||
/// The run that produced this turn, when the caller knows it.
|
||||
///
|
||||
/// Threaded in for the same reason as `outcome_override`: the contribution
|
||||
/// path reconstructs a trace from a transcript after the fact, so it
|
||||
/// cannot derive an identity that only existed during generation. This is
|
||||
/// the key the logprob sidecar is written under, and supplying it is what
|
||||
/// lets a contribution carry confidence aggregates.
|
||||
///
|
||||
/// `None` simply means no aggregates — capture is off by default, so that
|
||||
/// is the ordinary case and not a degradation.
|
||||
pub run_id: Option<&'a str>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -160,10 +171,15 @@ impl TraceClientHost {
|
||||
)),
|
||||
credit_account_ref: None,
|
||||
};
|
||||
let envelope = self
|
||||
let mut envelope = self
|
||||
.build_envelope_from_capture_turns(&turns, options, Some(persisted_outcome))
|
||||
.await
|
||||
.context("failed to redact autonomous trace")?;
|
||||
// Reduce whatever the logprob sidecar captured for this run into the
|
||||
// four numbers the envelope carries. The raw distributions stay on
|
||||
// disk: they cannot cross the ingest boundary and they are more
|
||||
// sensitive than the text they describe.
|
||||
envelope.training_dynamics = trace::training_dynamics_for_run(request.run_id);
|
||||
|
||||
match trace::trace_autonomous_eligibility(&envelope, request.policy) {
|
||||
trace::TraceQueueEligibility::Submit => Ok(
|
||||
@@ -484,6 +500,101 @@ mod tests {
|
||||
assert_eq!(parsed_outcome.task_success, TaskSuccess::Failure);
|
||||
}
|
||||
|
||||
/// The join the whole feature depends on: the model gateway writes the
|
||||
/// sidecar keyed by run id during generation, and the contribution path —
|
||||
/// which rebuilds a trace from a transcript afterwards and has no identity
|
||||
/// of its own — finds it again by that same key.
|
||||
#[tokio::test]
|
||||
async fn a_captured_run_contributes_confidence_aggregates() {
|
||||
let dir = std::env::temp_dir().join(format!("ironclaw-wiring-{}", uuid::Uuid::new_v4()));
|
||||
// SAFETY: the sidecar directory is read through the same env helper the
|
||||
// gateway writes through; this test owns the value for its duration.
|
||||
unsafe {
|
||||
std::env::set_var(ironclaw_llm::logprob_sidecar::LOGPROB_SIDECAR_DIR_ENV, &dir);
|
||||
}
|
||||
|
||||
let run_id = "run-wiring-test";
|
||||
ironclaw_llm::logprob_sidecar::append(
|
||||
&dir,
|
||||
Some(run_id),
|
||||
Some("turn-1"),
|
||||
"qwen3-30b",
|
||||
true,
|
||||
&[
|
||||
ironclaw_llm::logprob_sidecar::TokenLogprob {
|
||||
token: "a".into(),
|
||||
logprob: 0.9f32.ln(),
|
||||
top_logprobs: vec![],
|
||||
},
|
||||
ironclaw_llm::logprob_sidecar::TokenLogprob {
|
||||
token: "b".into(),
|
||||
logprob: 0.85f32.ln(),
|
||||
top_logprobs: vec![],
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
let policy = enabled_policy();
|
||||
let messages = vec![msg("user", "hello"), msg("assistant", "hi")];
|
||||
let envelope = TraceClientHost
|
||||
.build_autonomous_envelope_from_messages(TraceClientAutonomousCaptureRequest {
|
||||
scope: TraceClientScope::user("user-123"),
|
||||
channel: TraceChannel::Web,
|
||||
messages: &messages,
|
||||
policy: &policy,
|
||||
max_turns: 5,
|
||||
outcome_override: None,
|
||||
run_id: Some(run_id),
|
||||
})
|
||||
.await
|
||||
.expect("capture succeeds")
|
||||
.expect("eligible envelope");
|
||||
|
||||
let signals = envelope
|
||||
.training_dynamics
|
||||
.expect("a captured run contributes aggregates");
|
||||
let mean = signals.mean_confidence.expect("mean present");
|
||||
assert!(
|
||||
(mean - 0.875).abs() < 1e-3,
|
||||
"expected the mean of the captured probabilities, got {mean}"
|
||||
);
|
||||
assert_eq!(
|
||||
signals.cartography_bucket,
|
||||
Some(crate::contribution::CartographyBucket::Easy)
|
||||
);
|
||||
assert!(
|
||||
signals.correctness.is_none(),
|
||||
"correctness needs an outcome signal, not confidence"
|
||||
);
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var(ironclaw_llm::logprob_sidecar::LOGPROB_SIDECAR_DIR_ENV);
|
||||
}
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
/// Capture is off by default, so this is the ordinary path and it must not
|
||||
/// invent signals.
|
||||
#[tokio::test]
|
||||
async fn a_run_without_capture_contributes_no_aggregates() {
|
||||
let policy = enabled_policy();
|
||||
let messages = vec![msg("user", "hello"), msg("assistant", "hi")];
|
||||
let envelope = TraceClientHost
|
||||
.build_autonomous_envelope_from_messages(TraceClientAutonomousCaptureRequest {
|
||||
scope: TraceClientScope::user("user-123"),
|
||||
channel: TraceChannel::Web,
|
||||
messages: &messages,
|
||||
policy: &policy,
|
||||
max_turns: 5,
|
||||
outcome_override: None,
|
||||
run_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("capture succeeds")
|
||||
.expect("eligible envelope");
|
||||
assert!(envelope.training_dynamics.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn autonomous_capture_uses_scoped_client_identity() {
|
||||
let policy = enabled_policy();
|
||||
@@ -498,6 +609,7 @@ mod tests {
|
||||
policy: &policy,
|
||||
max_turns: 5,
|
||||
outcome_override: None,
|
||||
run_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("capture succeeds")
|
||||
@@ -530,6 +642,7 @@ mod tests {
|
||||
policy: &policy,
|
||||
max_turns: 5,
|
||||
outcome_override: None,
|
||||
run_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("policy skip is not an error");
|
||||
@@ -552,6 +665,7 @@ mod tests {
|
||||
policy: &policy,
|
||||
max_turns: 5,
|
||||
outcome_override: None,
|
||||
run_id: None,
|
||||
})
|
||||
.await
|
||||
.expect("capture evaluates");
|
||||
|
||||
Reference in New Issue
Block a user