diff --git a/kaiwu/core/orchestrator.py b/kaiwu/core/orchestrator.py index 1de6b9f..6246594 100644 --- a/kaiwu/core/orchestrator.py +++ b/kaiwu/core/orchestrator.py @@ -241,6 +241,8 @@ class PipelineOrchestrator: # 错误类型追踪 ctx._errors_encountered = [] + # 详细attempt记录（供trajectory_collector使用） + ctx._trajectory_attempts = [] # 每次顶层任务重置manifest self._manifest.clear() @@ -459,6 +461,13 @@ class PipelineOrchestrator: } ctx.attempt_history.append(attempt_record) + # ── Trajectory详细attempt记录（诊断用） ── + try: + _traj_attempt = self._build_trajectory_attempt(ctx, current_error_type) + ctx._trajectory_attempts.append(_traj_attempt) + except Exception: + pass # 记录失败不影响主流程 + # 快速熔断：语法错误重试无效 # syntax熔断按tier区分：SMALL立刻熔断，MEDIUM/LARGE多给一次 _syntax_max = 1 if self._model_tier == ModelTier.SMALL else 2 @@ -1235,3 +1244,83 @@ class PipelineOrchestrator: ) except Exception: return ctx.gap or Gap(GapType.UNKNOWN, 0.3, [], [], "", "") + + def _build_trajectory_attempt(self, ctx: TaskContext, error_type: str) -> dict: + """构建单次attempt的详细诊断记录（供trajectory.json使用）。 + + 所有字段try-except保护，单字段失败不影响其他字段。 + 字符串字段截断到合理长度（≤500字）。 + """ + attempt = {"attempt": ctx.retry_count - 1} # retry_count已+1，这里记录的是刚完成的attempt + + # LLM输入输出（从audit logger的llm_calls取最近一条） + try: + llm_calls = self._audit._llm_calls + if llm_calls: + last_call = llm_calls[-1] + attempt["llm_prompt_tail"] = (last_call.get("prompt_preview", "") or "")[-400:] + attempt["llm_raw_output"] = (last_call.get("raw_output", "") or "")[:400] + attempt["llm_caller"] = last_call.get("caller", "") + else: + attempt["llm_prompt_tail"] = "" + attempt["llm_raw_output"] = "" + attempt["llm_caller"] = "" + except Exception: + attempt["llm_prompt_tail"] = "" + attempt["llm_raw_output"] = "" + attempt["llm_caller"] = "" + + # patch生成结果 + try: + patches = (ctx.generator_output or {}).get("patches", []) + attempt["patches_count"] = len(patches) + except Exception: + attempt["patches_count"] = 0 + + # patch apply结果 + try: + v = ctx.verifier_output or {} + if v.get("error_type") == "patch_apply": + attempt["patch_apply_ok"] = False + attempt["patch_apply_error"] = (v.get("error_message", "") or v.get("error_detail", ""))[:300] + elif v: + # verifier跑到了测试阶段，说明apply成功了 + attempt["patch_apply_ok"] = True + attempt["patch_apply_error"] = "" + else: + attempt["patch_apply_ok"] = False + attempt["patch_apply_error"] = "no verifier output" + except Exception: + attempt["patch_apply_ok"] = False + attempt["patch_apply_error"] = "" + + # 修改行数 + try: + patches = (ctx.generator_output or {}).get("patches", []) + modified_lines = 0 + for p in patches: + orig = p.get("original", "") or "" + mod = p.get("modified", "") or p.get("content", "") or "" + modified_lines += abs(len(mod.splitlines()) - len(orig.splitlines())) + attempt["modified_lines"] = modified_lines + except Exception: + attempt["modified_lines"] = 0 + + # verifier测试结果 + try: + v = ctx.verifier_output or {} + attempt["tests_passed"] = v.get("tests_passed", 0) + attempt["tests_total"] = v.get("tests_total", 0) + attempt["test_output_tail"] = (v.get("error_detail", "") or "")[-300:] + except Exception: + attempt["tests_passed"] = 0 + attempt["tests_total"] = 0 + attempt["test_output_tail"] = "" + + # 错误类型 + try: + attempt["error_type"] = error_type or "" + except Exception: + attempt["error_type"] = "" + + return attempt diff --git a/kaiwu/flywheel/trajectory_collector.py b/kaiwu/flywheel/trajectory_collector.py index 195da20..b473465 100644 --- a/kaiwu/flywheel/trajectory_collector.py +++ b/kaiwu/flywheel/trajectory_collector.py @@ -35,6 +35,8 @@ class TaskTrajectory: timestamp: str = "" search_triggered: bool = False project_hash: str = "" + # 每次attempt的详细记录（诊断用） + attempts: list[dict] = field(default_factory=list) class TrajectoryCollector: @@ -66,6 +68,9 @@ class TrajectoryCollector: if ctx.generator_output and "patches" in ctx.generator_output: files_modified = [p.get("file", "") for p in ctx.generator_output["patches"] if p.get("file")] + # 收集详细attempts记录（从ctx._trajectory_attempts，orchestrator填充） + attempts = getattr(ctx, '_trajectory_attempts', []) + traj = TaskTrajectory( task_id=str(uuid.uuid4()), user_input=ctx.user_input, @@ -85,6 +90,7 @@ class TrajectoryCollector: timestamp=datetime.now(timezone.utc).isoformat(), search_triggered=ctx.search_triggered, project_hash=hashlib.sha256(ctx.project_root.encode()).hexdigest()[:16], + attempts=attempts, ) path = os.path.join(self._dir, f"{traj.task_id}.json") diff --git a/tests/test_audit_model.py b/tests/test_audit_model.py index 501d77f..4db84f0 100644 --- a/tests/test_audit_model.py +++ b/tests/test_audit_model.py @@ -21,29 +21,34 @@ class TestAuditLogger(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tmpdir) - @patch("kaiwu.audit.logger.LOGS_DIR") - def test_write_creates_log(self, mock_dir): - mock_dir.__class__ = type(self.logs_dir) + def test_write_creates_log(self): from kaiwu.audit.logger import AuditLogger - with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir): + success_dir = self.logs_dir / "success" + failed_dir = self.logs_dir / "failed" + with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \ + patch("kaiwu.audit.logger.LOGS_FAILED", failed_dir): logger = AuditLogger() logger.start() logger.log("gate", "locator_repair | 难度：easy") logger.log("locator", "读取 test.py") - # Mock context + # Mock context — need real values for fields accessed by write() ctx = MagicMock() ctx.user_input = "修复login函数" ctx.gate_result = {"expert_type": "locator_repair", "difficulty": "easy"} ctx.generator_output = {"patches": [{"file": "test.py", "original": "old", "modified": "new"}]} - ctx.verifier_output = {"tests_passed": 3, "tests_total": 3} + ctx.verifier_output = {"tests_passed": 3, "tests_total": 3, "passed": True, "error_type": "", "structured_failures": []} ctx.retry_count = 0 ctx.search_triggered = False + ctx.gap = None + ctx.locator_output = None + ctx.routing_source = "" + ctx.attempt_history = [] logger.write(ctx, 5.2, True, "qwen3:8b") # Verify log file created - logs = list(self.logs_dir.glob("*.json")) + logs = list(success_dir.glob("*.json")) assert len(logs) == 1 data = json.loads(logs[0].read_text(encoding="utf-8")) @@ -53,46 +58,56 @@ class TestAuditLogger(unittest.TestCase): assert len(data["events"]) == 2 assert data["files_modified"] == ["test.py"] - @patch("kaiwu.audit.logger.LOGS_DIR") - def test_list_logs(self, mock_dir): + def test_list_logs(self): from kaiwu.audit.logger import list_logs - with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir): - self.logs_dir.mkdir(parents=True, exist_ok=True) + success_dir = self.logs_dir / "success" + failed_dir = self.logs_dir / "failed" + legacy_dir = self.logs_dir / "legacy" + with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \ + patch("kaiwu.audit.logger.LOGS_FAILED", failed_dir), \ + patch("kaiwu.audit.logger.LOGS_LEGACY", legacy_dir): + success_dir.mkdir(parents=True, exist_ok=True) # Write 3 log files with distinct names for i in range(3): record = {"task": f"task {i}", "success": True, "elapsed_s": 1.0, "timestamp": f"2026-05-06T10:00:0{i}", "model": "test"} - (self.logs_dir / f"2026-05-06_10000{i}_codegen.json").write_text( + (success_dir / f"2026-05-06_10000{i}_codegen.json").write_text( json.dumps(record), encoding="utf-8" ) logs = list_logs(limit=10) assert len(logs) == 3 - @patch("kaiwu.audit.logger.LOGS_DIR") - def test_clear_logs(self, mock_dir): + def test_clear_logs(self): from kaiwu.audit.logger import AuditLogger, clear_logs - with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir): - self.logs_dir.mkdir(parents=True, exist_ok=True) - (self.logs_dir / "test.json").write_text("{}", encoding="utf-8") + success_dir = self.logs_dir / "success" + failed_dir = self.logs_dir / "failed" + legacy_dir = self.logs_dir / "legacy" + with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \ + patch("kaiwu.audit.logger.LOGS_FAILED", failed_dir), \ + patch("kaiwu.audit.logger.LOGS_LEGACY", legacy_dir): + success_dir.mkdir(parents=True, exist_ok=True) + (success_dir / "test.json").write_text("{}", encoding="utf-8") count = clear_logs() assert count == 1 - assert len(list(self.logs_dir.glob("*.json"))) == 0 + assert len(list(success_dir.glob("*.json"))) == 0 - @patch("kaiwu.audit.logger.LOGS_DIR") - def test_max_logs_cleanup(self, mock_dir): + def test_max_logs_cleanup(self): from kaiwu.audit.logger import AuditLogger, MAX_LOGS - with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir): - self.logs_dir.mkdir(parents=True, exist_ok=True) + success_dir = self.logs_dir / "success" + with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \ + patch("kaiwu.audit.logger.LOGS_FAILED", self.logs_dir / "failed"), \ + patch("kaiwu.audit.logger.LOGS_LEGACY", self.logs_dir / "legacy"): + success_dir.mkdir(parents=True, exist_ok=True) # Create MAX_LOGS + 5 files for i in range(MAX_LOGS + 5): - (self.logs_dir / f"2026-01-01_{i:06d}_test.json").write_text("{}", encoding="utf-8") + (success_dir / f"2026-01-01_{i:06d}_test.json").write_text("{}", encoding="utf-8") al = AuditLogger() - al._cleanup() + al._cleanup(success_dir) - remaining = list(self.logs_dir.glob("*.json")) + remaining = list(success_dir.glob("*.json")) assert len(remaining) == MAX_LOGS