V6 expert generation validation: gemma4:e2b 3/3 PASS

- gemma3:4b: 1/3 (small model struggles with complex JSON generation)
- gemma4:e2b: 3/3 (FastAPI CRUD, pytest fixture, DB migration all pass)
- All generated experts have valid structure, keywords, and pipelines

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Val-sss
2026-04-27 00:22:31 +08:00
parent 5668419bdf
commit 235ecd2928
2 changed files with 106 additions and 2 deletions

View File

@@ -57,7 +57,7 @@ MVP 流水线 + 6 步搜索增强 + 专家注册表 + 3 层记忆 + 飞轮自动
| V3 Locator精度 | 文件级 90%,函数级 20% | 函数级已加 few-shot 优化,待更大模型验证 |
| V4 搜索模块 | 意图4/4, DDG 4/4, Fetch 3/4, 压缩4/4 | trafilatura+bs4, 耗时略超15s(LLM瓶颈) |
| V5 AST Locator | 脚本就绪,待运行 | A组(LLM猜测) vs B组(tree-sitter stub) |
| V6 专家生成质量 | 脚本就绪,待运行 | 3组×5轨迹需要Ollama在线 |
| V6 专家生成质量 | gemma3:4b 1/3, gemma4:e2b 3/3 PASS | 小模型JSON生成弱大模型全过 |
| E2E 单文件 | 通过 | gemma3:4b 5.7s / gemma4:e2b 64.9s5/5 测试 |
| E2E 多文件 | 通过 | gemma3:4b 7.7spassword leak 跨2文件3/3 测试 |
| gemma4:e2b Gate | 100% 类型准确率(含 office | 比 gemma3:4b 的 67% 大幅提升,但慢 10x |
@@ -270,7 +270,7 @@ kaiwu/
- [x] Windows cmd原生验证Python import + pytest 24/24 通过)
- [x] 红线约束代码review10/10 CORE 全部 PASS
- [ ] V5 AST Locator实际验证需要tree-sitter + Ollama
- [ ] V6 专家生成质量实际验证(需要Ollama
- [x] V6 专家生成质量验证(gemma4:e2b 3/3 PASSgemma3:4b 1/3
- [ ] 预置专家20任务验证Step 16
### 已知限制

View File

@@ -0,0 +1,104 @@
{
"status": "completed",
"model": "gemma4:e2b",
"groups_passed": 3,
"groups_total": 3,
"overall_pass": true,
"groups": {
"fastapi_crud": {
"label": "FastAPI CRUD endpoint",
"generated": true,
"expert_name": "FastAPICodeGeneratorExpert",
"expert_keywords": [
"FastAPI",
"CRUD接口",
"API实现",
"数据管理",
"模型生成"
],
"expert_pipeline": [
"generator",
"verifier"
],
"system_prompt_len": 604,
"structure_checks": {
"has_name": true,
"has_trigger_keywords": true,
"has_system_prompt": true,
"has_pipeline": true,
"pipeline_valid": true,
"system_prompt_nonempty": true,
"system_prompt_has_content": true,
"confidence_reasonable": true,
"all_passed": true
},
"keyword_coverage": 0.6,
"elapsed_s": 54.7,
"passed": true
},
"pytest_fixture": {
"label": "pytest fixture",
"generated": true,
"expert_name": "PytestFixtureGenerator",
"expert_keywords": [
"pytest fixture",
"pytest conftest",
"test setup",
"fixture creation",
"test teardown",
"mocking fixture"
],
"expert_pipeline": [
"generator",
"verifier"
],
"system_prompt_len": 592,
"structure_checks": {
"has_name": true,
"has_trigger_keywords": true,
"has_system_prompt": true,
"has_pipeline": true,
"pipeline_valid": true,
"system_prompt_nonempty": true,
"system_prompt_has_content": true,
"confidence_reasonable": true,
"all_passed": true
},
"keyword_coverage": 1.0,
"elapsed_s": 31.1,
"passed": true
},
"db_migration": {
"label": "database migration",
"generated": true,
"expert_name": "DatabaseMigrationCodegenExpert",
"expert_keywords": [
"写迁移脚本",
"Alembic",
"数据库迁移",
"schema 变更",
"表结构修改",
"模型迁移"
],
"expert_pipeline": [
"generator",
"verifier"
],
"system_prompt_len": 592,
"structure_checks": {
"has_name": true,
"has_trigger_keywords": true,
"has_system_prompt": true,
"has_pipeline": true,
"pipeline_valid": true,
"system_prompt_nonempty": true,
"system_prompt_has_content": true,
"confidence_reasonable": true,
"all_passed": true
},
"keyword_coverage": 0.4,
"elapsed_s": 32.6,
"passed": true
}
}
}