mirror of
https://github.com/val1813/kwcode.git
synced 2026-09-03 06:34:30 +08:00
V6 expert generation validation: gemma4:e2b 3/3 PASS
- gemma3:4b: 1/3 (small model struggles with complex JSON generation) - gemma4:e2b: 3/3 (FastAPI CRUD, pytest fixture, DB migration all pass) - All generated experts have valid structure, keywords, and pipelines Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -57,7 +57,7 @@ MVP 流水线 + 6 步搜索增强 + 专家注册表 + 3 层记忆 + 飞轮自动
|
||||
| V3 Locator精度 | 文件级 90%,函数级 20% | 函数级已加 few-shot 优化,待更大模型验证 |
|
||||
| V4 搜索模块 | 意图4/4, DDG 4/4, Fetch 3/4, 压缩4/4 | trafilatura+bs4, 耗时略超15s(LLM瓶颈) |
|
||||
| V5 AST Locator | 脚本就绪,待运行 | A组(LLM猜测) vs B组(tree-sitter stub) |
|
||||
| V6 专家生成质量 | 脚本就绪,待运行 | 3组×5轨迹,需要Ollama在线 |
|
||||
| V6 专家生成质量 | gemma3:4b 1/3, gemma4:e2b 3/3 PASS | 小模型JSON生成弱,大模型全过 |
|
||||
| E2E 单文件 | 通过 | gemma3:4b 5.7s / gemma4:e2b 64.9s,5/5 测试 |
|
||||
| E2E 多文件 | 通过 | gemma3:4b 7.7s,password leak 跨2文件,3/3 测试 |
|
||||
| gemma4:e2b Gate | 100% 类型准确率(含 office) | 比 gemma3:4b 的 67% 大幅提升,但慢 10x |
|
||||
@@ -270,7 +270,7 @@ kaiwu/
|
||||
- [x] Windows cmd原生验证(Python import + pytest 24/24 通过)
|
||||
- [x] 红线约束代码review(10/10 CORE 全部 PASS)
|
||||
- [ ] V5 AST Locator实际验证(需要tree-sitter + Ollama)
|
||||
- [ ] V6 专家生成质量实际验证(需要Ollama)
|
||||
- [x] V6 专家生成质量验证(gemma4:e2b 3/3 PASS,gemma3:4b 1/3)
|
||||
- [ ] 预置专家20任务验证(Step 16)
|
||||
|
||||
### 已知限制
|
||||
|
||||
104
kaiwu/validation/v6_conclusion.json
Normal file
104
kaiwu/validation/v6_conclusion.json
Normal file
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"status": "completed",
|
||||
"model": "gemma4:e2b",
|
||||
"groups_passed": 3,
|
||||
"groups_total": 3,
|
||||
"overall_pass": true,
|
||||
"groups": {
|
||||
"fastapi_crud": {
|
||||
"label": "FastAPI CRUD endpoint",
|
||||
"generated": true,
|
||||
"expert_name": "FastAPICodeGeneratorExpert",
|
||||
"expert_keywords": [
|
||||
"FastAPI",
|
||||
"CRUD接口",
|
||||
"API实现",
|
||||
"数据管理",
|
||||
"模型生成"
|
||||
],
|
||||
"expert_pipeline": [
|
||||
"generator",
|
||||
"verifier"
|
||||
],
|
||||
"system_prompt_len": 604,
|
||||
"structure_checks": {
|
||||
"has_name": true,
|
||||
"has_trigger_keywords": true,
|
||||
"has_system_prompt": true,
|
||||
"has_pipeline": true,
|
||||
"pipeline_valid": true,
|
||||
"system_prompt_nonempty": true,
|
||||
"system_prompt_has_content": true,
|
||||
"confidence_reasonable": true,
|
||||
"all_passed": true
|
||||
},
|
||||
"keyword_coverage": 0.6,
|
||||
"elapsed_s": 54.7,
|
||||
"passed": true
|
||||
},
|
||||
"pytest_fixture": {
|
||||
"label": "pytest fixture",
|
||||
"generated": true,
|
||||
"expert_name": "PytestFixtureGenerator",
|
||||
"expert_keywords": [
|
||||
"pytest fixture",
|
||||
"pytest conftest",
|
||||
"test setup",
|
||||
"fixture creation",
|
||||
"test teardown",
|
||||
"mocking fixture"
|
||||
],
|
||||
"expert_pipeline": [
|
||||
"generator",
|
||||
"verifier"
|
||||
],
|
||||
"system_prompt_len": 592,
|
||||
"structure_checks": {
|
||||
"has_name": true,
|
||||
"has_trigger_keywords": true,
|
||||
"has_system_prompt": true,
|
||||
"has_pipeline": true,
|
||||
"pipeline_valid": true,
|
||||
"system_prompt_nonempty": true,
|
||||
"system_prompt_has_content": true,
|
||||
"confidence_reasonable": true,
|
||||
"all_passed": true
|
||||
},
|
||||
"keyword_coverage": 1.0,
|
||||
"elapsed_s": 31.1,
|
||||
"passed": true
|
||||
},
|
||||
"db_migration": {
|
||||
"label": "database migration",
|
||||
"generated": true,
|
||||
"expert_name": "DatabaseMigrationCodegenExpert",
|
||||
"expert_keywords": [
|
||||
"写迁移脚本",
|
||||
"Alembic",
|
||||
"数据库迁移",
|
||||
"schema 变更",
|
||||
"表结构修改",
|
||||
"模型迁移"
|
||||
],
|
||||
"expert_pipeline": [
|
||||
"generator",
|
||||
"verifier"
|
||||
],
|
||||
"system_prompt_len": 592,
|
||||
"structure_checks": {
|
||||
"has_name": true,
|
||||
"has_trigger_keywords": true,
|
||||
"has_system_prompt": true,
|
||||
"has_pipeline": true,
|
||||
"pipeline_valid": true,
|
||||
"system_prompt_nonempty": true,
|
||||
"system_prompt_has_content": true,
|
||||
"confidence_reasonable": true,
|
||||
"all_passed": true
|
||||
},
|
||||
"keyword_coverage": 0.4,
|
||||
"elapsed_s": 32.6,
|
||||
"passed": true
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user