mirror of
https://github.com/val1813/kwcode.git
synced 2026-09-03 06:34:30 +08:00
feat: add detailed attempts list to trajectory records
Each retry attempt now records llm_prompt_tail, llm_raw_output, llm_caller, patches_count, patch_apply_ok/error, modified_lines, tests_passed/total, test_output_tail, and error_type. This enables precise post-hoc diagnosis without re-running benchmarks. Also fixes test_audit_model.py to use correct module-level constants (LOGS_SUCCESS/LOGS_FAILED/LOGS_LEGACY instead of removed LOGS_DIR). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -241,6 +241,8 @@ class PipelineOrchestrator:
|
||||
|
||||
# 错误类型追踪
|
||||
ctx._errors_encountered = []
|
||||
# 详细attempt记录(供trajectory_collector使用)
|
||||
ctx._trajectory_attempts = []
|
||||
|
||||
# 每次顶层任务重置manifest
|
||||
self._manifest.clear()
|
||||
@@ -459,6 +461,13 @@ class PipelineOrchestrator:
|
||||
}
|
||||
ctx.attempt_history.append(attempt_record)
|
||||
|
||||
# ── Trajectory详细attempt记录(诊断用) ──
|
||||
try:
|
||||
_traj_attempt = self._build_trajectory_attempt(ctx, current_error_type)
|
||||
ctx._trajectory_attempts.append(_traj_attempt)
|
||||
except Exception:
|
||||
pass # 记录失败不影响主流程
|
||||
|
||||
# 快速熔断:语法错误重试无效
|
||||
# syntax熔断按tier区分:SMALL立刻熔断,MEDIUM/LARGE多给一次
|
||||
_syntax_max = 1 if self._model_tier == ModelTier.SMALL else 2
|
||||
@@ -1235,3 +1244,83 @@ class PipelineOrchestrator:
|
||||
)
|
||||
except Exception:
|
||||
return ctx.gap or Gap(GapType.UNKNOWN, 0.3, [], [], "", "")
|
||||
|
||||
def _build_trajectory_attempt(self, ctx: TaskContext, error_type: str) -> dict:
|
||||
"""构建单次attempt的详细诊断记录(供trajectory.json使用)。
|
||||
|
||||
所有字段try-except保护,单字段失败不影响其他字段。
|
||||
字符串字段截断到合理长度(≤500字)。
|
||||
"""
|
||||
attempt = {"attempt": ctx.retry_count - 1} # retry_count已+1,这里记录的是刚完成的attempt
|
||||
|
||||
# LLM输入输出(从audit logger的llm_calls取最近一条)
|
||||
try:
|
||||
llm_calls = self._audit._llm_calls
|
||||
if llm_calls:
|
||||
last_call = llm_calls[-1]
|
||||
attempt["llm_prompt_tail"] = (last_call.get("prompt_preview", "") or "")[-400:]
|
||||
attempt["llm_raw_output"] = (last_call.get("raw_output", "") or "")[:400]
|
||||
attempt["llm_caller"] = last_call.get("caller", "")
|
||||
else:
|
||||
attempt["llm_prompt_tail"] = ""
|
||||
attempt["llm_raw_output"] = ""
|
||||
attempt["llm_caller"] = ""
|
||||
except Exception:
|
||||
attempt["llm_prompt_tail"] = ""
|
||||
attempt["llm_raw_output"] = ""
|
||||
attempt["llm_caller"] = ""
|
||||
|
||||
# patch生成结果
|
||||
try:
|
||||
patches = (ctx.generator_output or {}).get("patches", [])
|
||||
attempt["patches_count"] = len(patches)
|
||||
except Exception:
|
||||
attempt["patches_count"] = 0
|
||||
|
||||
# patch apply结果
|
||||
try:
|
||||
v = ctx.verifier_output or {}
|
||||
if v.get("error_type") == "patch_apply":
|
||||
attempt["patch_apply_ok"] = False
|
||||
attempt["patch_apply_error"] = (v.get("error_message", "") or v.get("error_detail", ""))[:300]
|
||||
elif v:
|
||||
# verifier跑到了测试阶段,说明apply成功了
|
||||
attempt["patch_apply_ok"] = True
|
||||
attempt["patch_apply_error"] = ""
|
||||
else:
|
||||
attempt["patch_apply_ok"] = False
|
||||
attempt["patch_apply_error"] = "no verifier output"
|
||||
except Exception:
|
||||
attempt["patch_apply_ok"] = False
|
||||
attempt["patch_apply_error"] = ""
|
||||
|
||||
# 修改行数
|
||||
try:
|
||||
patches = (ctx.generator_output or {}).get("patches", [])
|
||||
modified_lines = 0
|
||||
for p in patches:
|
||||
orig = p.get("original", "") or ""
|
||||
mod = p.get("modified", "") or p.get("content", "") or ""
|
||||
modified_lines += abs(len(mod.splitlines()) - len(orig.splitlines()))
|
||||
attempt["modified_lines"] = modified_lines
|
||||
except Exception:
|
||||
attempt["modified_lines"] = 0
|
||||
|
||||
# verifier测试结果
|
||||
try:
|
||||
v = ctx.verifier_output or {}
|
||||
attempt["tests_passed"] = v.get("tests_passed", 0)
|
||||
attempt["tests_total"] = v.get("tests_total", 0)
|
||||
attempt["test_output_tail"] = (v.get("error_detail", "") or "")[-300:]
|
||||
except Exception:
|
||||
attempt["tests_passed"] = 0
|
||||
attempt["tests_total"] = 0
|
||||
attempt["test_output_tail"] = ""
|
||||
|
||||
# 错误类型
|
||||
try:
|
||||
attempt["error_type"] = error_type or ""
|
||||
except Exception:
|
||||
attempt["error_type"] = ""
|
||||
|
||||
return attempt
|
||||
|
||||
@@ -35,6 +35,8 @@ class TaskTrajectory:
|
||||
timestamp: str = ""
|
||||
search_triggered: bool = False
|
||||
project_hash: str = ""
|
||||
# 每次attempt的详细记录(诊断用)
|
||||
attempts: list[dict] = field(default_factory=list)
|
||||
|
||||
|
||||
class TrajectoryCollector:
|
||||
@@ -66,6 +68,9 @@ class TrajectoryCollector:
|
||||
if ctx.generator_output and "patches" in ctx.generator_output:
|
||||
files_modified = [p.get("file", "") for p in ctx.generator_output["patches"] if p.get("file")]
|
||||
|
||||
# 收集详细attempts记录(从ctx._trajectory_attempts,orchestrator填充)
|
||||
attempts = getattr(ctx, '_trajectory_attempts', [])
|
||||
|
||||
traj = TaskTrajectory(
|
||||
task_id=str(uuid.uuid4()),
|
||||
user_input=ctx.user_input,
|
||||
@@ -85,6 +90,7 @@ class TrajectoryCollector:
|
||||
timestamp=datetime.now(timezone.utc).isoformat(),
|
||||
search_triggered=ctx.search_triggered,
|
||||
project_hash=hashlib.sha256(ctx.project_root.encode()).hexdigest()[:16],
|
||||
attempts=attempts,
|
||||
)
|
||||
|
||||
path = os.path.join(self._dir, f"{traj.task_id}.json")
|
||||
|
||||
@@ -21,29 +21,34 @@ class TestAuditLogger(unittest.TestCase):
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmpdir)
|
||||
|
||||
@patch("kaiwu.audit.logger.LOGS_DIR")
|
||||
def test_write_creates_log(self, mock_dir):
|
||||
mock_dir.__class__ = type(self.logs_dir)
|
||||
def test_write_creates_log(self):
|
||||
from kaiwu.audit.logger import AuditLogger
|
||||
with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir):
|
||||
success_dir = self.logs_dir / "success"
|
||||
failed_dir = self.logs_dir / "failed"
|
||||
with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \
|
||||
patch("kaiwu.audit.logger.LOGS_FAILED", failed_dir):
|
||||
logger = AuditLogger()
|
||||
logger.start()
|
||||
logger.log("gate", "locator_repair | 难度:easy")
|
||||
logger.log("locator", "读取 test.py")
|
||||
|
||||
# Mock context
|
||||
# Mock context — need real values for fields accessed by write()
|
||||
ctx = MagicMock()
|
||||
ctx.user_input = "修复login函数"
|
||||
ctx.gate_result = {"expert_type": "locator_repair", "difficulty": "easy"}
|
||||
ctx.generator_output = {"patches": [{"file": "test.py", "original": "old", "modified": "new"}]}
|
||||
ctx.verifier_output = {"tests_passed": 3, "tests_total": 3}
|
||||
ctx.verifier_output = {"tests_passed": 3, "tests_total": 3, "passed": True, "error_type": "", "structured_failures": []}
|
||||
ctx.retry_count = 0
|
||||
ctx.search_triggered = False
|
||||
ctx.gap = None
|
||||
ctx.locator_output = None
|
||||
ctx.routing_source = ""
|
||||
ctx.attempt_history = []
|
||||
|
||||
logger.write(ctx, 5.2, True, "qwen3:8b")
|
||||
|
||||
# Verify log file created
|
||||
logs = list(self.logs_dir.glob("*.json"))
|
||||
logs = list(success_dir.glob("*.json"))
|
||||
assert len(logs) == 1
|
||||
|
||||
data = json.loads(logs[0].read_text(encoding="utf-8"))
|
||||
@@ -53,46 +58,56 @@ class TestAuditLogger(unittest.TestCase):
|
||||
assert len(data["events"]) == 2
|
||||
assert data["files_modified"] == ["test.py"]
|
||||
|
||||
@patch("kaiwu.audit.logger.LOGS_DIR")
|
||||
def test_list_logs(self, mock_dir):
|
||||
def test_list_logs(self):
|
||||
from kaiwu.audit.logger import list_logs
|
||||
with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir):
|
||||
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
success_dir = self.logs_dir / "success"
|
||||
failed_dir = self.logs_dir / "failed"
|
||||
legacy_dir = self.logs_dir / "legacy"
|
||||
with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \
|
||||
patch("kaiwu.audit.logger.LOGS_FAILED", failed_dir), \
|
||||
patch("kaiwu.audit.logger.LOGS_LEGACY", legacy_dir):
|
||||
success_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Write 3 log files with distinct names
|
||||
for i in range(3):
|
||||
record = {"task": f"task {i}", "success": True, "elapsed_s": 1.0,
|
||||
"timestamp": f"2026-05-06T10:00:0{i}", "model": "test"}
|
||||
(self.logs_dir / f"2026-05-06_10000{i}_codegen.json").write_text(
|
||||
(success_dir / f"2026-05-06_10000{i}_codegen.json").write_text(
|
||||
json.dumps(record), encoding="utf-8"
|
||||
)
|
||||
|
||||
logs = list_logs(limit=10)
|
||||
assert len(logs) == 3
|
||||
|
||||
@patch("kaiwu.audit.logger.LOGS_DIR")
|
||||
def test_clear_logs(self, mock_dir):
|
||||
def test_clear_logs(self):
|
||||
from kaiwu.audit.logger import AuditLogger, clear_logs
|
||||
with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir):
|
||||
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
(self.logs_dir / "test.json").write_text("{}", encoding="utf-8")
|
||||
success_dir = self.logs_dir / "success"
|
||||
failed_dir = self.logs_dir / "failed"
|
||||
legacy_dir = self.logs_dir / "legacy"
|
||||
with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \
|
||||
patch("kaiwu.audit.logger.LOGS_FAILED", failed_dir), \
|
||||
patch("kaiwu.audit.logger.LOGS_LEGACY", legacy_dir):
|
||||
success_dir.mkdir(parents=True, exist_ok=True)
|
||||
(success_dir / "test.json").write_text("{}", encoding="utf-8")
|
||||
count = clear_logs()
|
||||
assert count == 1
|
||||
assert len(list(self.logs_dir.glob("*.json"))) == 0
|
||||
assert len(list(success_dir.glob("*.json"))) == 0
|
||||
|
||||
@patch("kaiwu.audit.logger.LOGS_DIR")
|
||||
def test_max_logs_cleanup(self, mock_dir):
|
||||
def test_max_logs_cleanup(self):
|
||||
from kaiwu.audit.logger import AuditLogger, MAX_LOGS
|
||||
with patch("kaiwu.audit.logger.LOGS_DIR", self.logs_dir):
|
||||
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
success_dir = self.logs_dir / "success"
|
||||
with patch("kaiwu.audit.logger.LOGS_SUCCESS", success_dir), \
|
||||
patch("kaiwu.audit.logger.LOGS_FAILED", self.logs_dir / "failed"), \
|
||||
patch("kaiwu.audit.logger.LOGS_LEGACY", self.logs_dir / "legacy"):
|
||||
success_dir.mkdir(parents=True, exist_ok=True)
|
||||
# Create MAX_LOGS + 5 files
|
||||
for i in range(MAX_LOGS + 5):
|
||||
(self.logs_dir / f"2026-01-01_{i:06d}_test.json").write_text("{}", encoding="utf-8")
|
||||
(success_dir / f"2026-01-01_{i:06d}_test.json").write_text("{}", encoding="utf-8")
|
||||
|
||||
al = AuditLogger()
|
||||
al._cleanup()
|
||||
al._cleanup(success_dir)
|
||||
|
||||
remaining = list(self.logs_dir.glob("*.json"))
|
||||
remaining = list(success_dir.glob("*.json"))
|
||||
assert len(remaining) == MAX_LOGS
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user