feat: add detailed attempts list to trajectory records

Each retry attempt now records llm_prompt_tail, llm_raw_output,
llm_caller, patches_count, patch_apply_ok/error, modified_lines,
tests_passed/total, test_output_tail, and error_type. This enables
precise post-hoc diagnosis without re-running benchmarks.

Also fixes test_audit_model.py to use correct module-level constants
(LOGS_SUCCESS/LOGS_FAILED/LOGS_LEGACY instead of removed LOGS_DIR).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Val-sss
2026-05-08 09:22:47 +08:00
parent 5290dd117c
commit 8874de74a3
3 changed files with 135 additions and 25 deletions

View File

@@ -241,6 +241,8 @@ class PipelineOrchestrator:
# 错误类型追踪
ctx._errors_encountered = []
# 详细attempt记录供trajectory_collector使用
ctx._trajectory_attempts = []
# 每次顶层任务重置manifest
self._manifest.clear()
@@ -459,6 +461,13 @@ class PipelineOrchestrator:
}
ctx.attempt_history.append(attempt_record)
# ── Trajectory详细attempt记录诊断用 ──
try:
_traj_attempt = self._build_trajectory_attempt(ctx, current_error_type)
ctx._trajectory_attempts.append(_traj_attempt)
except Exception:
pass # 记录失败不影响主流程
# 快速熔断:语法错误重试无效
# syntax熔断按tier区分SMALL立刻熔断MEDIUM/LARGE多给一次
_syntax_max = 1 if self._model_tier == ModelTier.SMALL else 2
@@ -1235,3 +1244,83 @@ class PipelineOrchestrator:
)
except Exception:
return ctx.gap or Gap(GapType.UNKNOWN, 0.3, [], [], "", "")
def _build_trajectory_attempt(self, ctx: TaskContext, error_type: str) -> dict:
"""构建单次attempt的详细诊断记录供trajectory.json使用
所有字段try-except保护单字段失败不影响其他字段。
字符串字段截断到合理长度≤500字
"""
attempt = {"attempt": ctx.retry_count - 1} # retry_count已+1这里记录的是刚完成的attempt
# LLM输入输出从audit logger的llm_calls取最近一条
try:
llm_calls = self._audit._llm_calls
if llm_calls:
last_call = llm_calls[-1]
attempt["llm_prompt_tail"] = (last_call.get("prompt_preview", "") or "")[-400:]
attempt["llm_raw_output"] = (last_call.get("raw_output", "") or "")[:400]
attempt["llm_caller"] = last_call.get("caller", "")
else:
attempt["llm_prompt_tail"] = ""
attempt["llm_raw_output"] = ""
attempt["llm_caller"] = ""
except Exception:
attempt["llm_prompt_tail"] = ""
attempt["llm_raw_output"] = ""
attempt["llm_caller"] = ""
# patch生成结果
try:
patches = (ctx.generator_output or {}).get("patches", [])
attempt["patches_count"] = len(patches)
except Exception:
attempt["patches_count"] = 0
# patch apply结果
try:
v = ctx.verifier_output or {}
if v.get("error_type") == "patch_apply":
attempt["patch_apply_ok"] = False
attempt["patch_apply_error"] = (v.get("error_message", "") or v.get("error_detail", ""))[:300]
elif v:
# verifier跑到了测试阶段说明apply成功了
attempt["patch_apply_ok"] = True
attempt["patch_apply_error"] = ""
else:
attempt["patch_apply_ok"] = False
attempt["patch_apply_error"] = "no verifier output"
except Exception:
attempt["patch_apply_ok"] = False
attempt["patch_apply_error"] = ""
# 修改行数
try:
patches = (ctx.generator_output or {}).get("patches", [])
modified_lines = 0
for p in patches:
orig = p.get("original", "") or ""
mod = p.get("modified", "") or p.get("content", "") or ""
modified_lines += abs(len(mod.splitlines()) - len(orig.splitlines()))
attempt["modified_lines"] = modified_lines
except Exception:
attempt["modified_lines"] = 0
# verifier测试结果
try:
v = ctx.verifier_output or {}
attempt["tests_passed"] = v.get("tests_passed", 0)
attempt["tests_total"] = v.get("tests_total", 0)
attempt["test_output_tail"] = (v.get("error_detail", "") or "")[-300:]
except Exception:
attempt["tests_passed"] = 0
attempt["tests_total"] = 0
attempt["test_output_tail"] = ""
# 错误类型
try:
attempt["error_type"] = error_type or ""
except Exception:
attempt["error_type"] = ""
return attempt

View File

@@ -35,6 +35,8 @@ class TaskTrajectory:
timestamp: str = ""
search_triggered: bool = False
project_hash: str = ""
# 每次attempt的详细记录诊断用
attempts: list[dict] = field(default_factory=list)
class TrajectoryCollector:
@@ -66,6 +68,9 @@ class TrajectoryCollector:
if ctx.generator_output and "patches" in ctx.generator_output:
files_modified = [p.get("file", "") for p in ctx.generator_output["patches"] if p.get("file")]
# 收集详细attempts记录从ctx._trajectory_attemptsorchestrator填充
attempts = getattr(ctx, '_trajectory_attempts', [])
traj = TaskTrajectory(
task_id=str(uuid.uuid4()),
user_input=ctx.user_input,
@@ -85,6 +90,7 @@ class TrajectoryCollector:
timestamp=datetime.now(timezone.utc).isoformat(),
search_triggered=ctx.search_triggered,
project_hash=hashlib.sha256(ctx.project_root.encode()).hexdigest()[:16],
attempts=attempts,
)
path = os.path.join(self._dir, f"{traj.task_id}.json")

View File

@@ -21,29 +21,34 @@ class TestAuditLogger(unittest.TestCase):
def tearDown(self):
shutil.rmtree(self.tmpdir)
@patch("kaiwu.audit.logger.LOGS_DIR")
def test_write_creates_log(self, mock_dir):
mock_dir.__class__ = type(self.logs_dir)
def test_write_creates_log(self):
from kaiwu.audit.logger import AuditLogger
with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir):
success_dir = self.logs_dir / "success"
failed_dir = self.logs_dir / "failed"
with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \
patch("kaiwu.audit.logger.LOGS_FAILED", failed_dir):
logger = AuditLogger()
logger.start()
logger.log("gate", "locator_repair | 难度easy")
logger.log("locator", "读取 test.py")
# Mock context
# Mock context — need real values for fields accessed by write()
ctx = MagicMock()
ctx.user_input = "修复login函数"
ctx.gate_result = {"expert_type": "locator_repair", "difficulty": "easy"}
ctx.generator_output = {"patches": [{"file": "test.py", "original": "old", "modified": "new"}]}
ctx.verifier_output = {"tests_passed": 3, "tests_total": 3}
ctx.verifier_output = {"tests_passed": 3, "tests_total": 3, "passed": True, "error_type": "", "structured_failures": []}
ctx.retry_count = 0
ctx.search_triggered = False
ctx.gap = None
ctx.locator_output = None
ctx.routing_source = ""
ctx.attempt_history = []
logger.write(ctx, 5.2, True, "qwen3:8b")
# Verify log file created
logs = list(self.logs_dir.glob("*.json"))
logs = list(success_dir.glob("*.json"))
assert len(logs) == 1
data = json.loads(logs[0].read_text(encoding="utf-8"))
@@ -53,46 +58,56 @@ class TestAuditLogger(unittest.TestCase):
assert len(data["events"]) == 2
assert data["files_modified"] == ["test.py"]
@patch("kaiwu.audit.logger.LOGS_DIR")
def test_list_logs(self, mock_dir):
def test_list_logs(self):
from kaiwu.audit.logger import list_logs
with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir):
self.logs_dir.mkdir(parents=True, exist_ok=True)
success_dir = self.logs_dir / "success"
failed_dir = self.logs_dir / "failed"
legacy_dir = self.logs_dir / "legacy"
with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \
patch("kaiwu.audit.logger.LOGS_FAILED", failed_dir), \
patch("kaiwu.audit.logger.LOGS_LEGACY", legacy_dir):
success_dir.mkdir(parents=True, exist_ok=True)
# Write 3 log files with distinct names
for i in range(3):
record = {"task": f"task {i}", "success": True, "elapsed_s": 1.0,
"timestamp": f"2026-05-06T10:00:0{i}", "model": "test"}
(self.logs_dir / f"2026-05-06_10000{i}_codegen.json").write_text(
(success_dir / f"2026-05-06_10000{i}_codegen.json").write_text(
json.dumps(record), encoding="utf-8"
)
logs = list_logs(limit=10)
assert len(logs) == 3
@patch("kaiwu.audit.logger.LOGS_DIR")
def test_clear_logs(self, mock_dir):
def test_clear_logs(self):
from kaiwu.audit.logger import AuditLogger, clear_logs
with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir):
self.logs_dir.mkdir(parents=True, exist_ok=True)
(self.logs_dir / "test.json").write_text("{}", encoding="utf-8")
success_dir = self.logs_dir / "success"
failed_dir = self.logs_dir / "failed"
legacy_dir = self.logs_dir / "legacy"
with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \
patch("kaiwu.audit.logger.LOGS_FAILED", failed_dir), \
patch("kaiwu.audit.logger.LOGS_LEGACY", legacy_dir):
success_dir.mkdir(parents=True, exist_ok=True)
(success_dir / "test.json").write_text("{}", encoding="utf-8")
count = clear_logs()
assert count == 1
assert len(list(self.logs_dir.glob("*.json"))) == 0
assert len(list(success_dir.glob("*.json"))) == 0
@patch("kaiwu.audit.logger.LOGS_DIR")
def test_max_logs_cleanup(self, mock_dir):
def test_max_logs_cleanup(self):
from kaiwu.audit.logger import AuditLogger, MAX_LOGS
with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir):
self.logs_dir.mkdir(parents=True, exist_ok=True)
success_dir = self.logs_dir / "success"
with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \
patch("kaiwu.audit.logger.LOGS_FAILED", self.logs_dir / "failed"), \
patch("kaiwu.audit.logger.LOGS_LEGACY", self.logs_dir / "legacy"):
success_dir.mkdir(parents=True, exist_ok=True)
# Create MAX_LOGS + 5 files
for i in range(MAX_LOGS + 5):
(self.logs_dir / f"2026-01-01_{i:06d}_test.json").write_text("{}", encoding="utf-8")
(success_dir / f"2026-01-01_{i:06d}_test.json").write_text("{}", encoding="utf-8")
al = AuditLogger()
al._cleanup()
al._cleanup(success_dir)
remaining = list(self.logs_dir.glob("*.json"))
remaining = list(success_dir.glob("*.json"))
assert len(remaining) == MAX_LOGS