diff --git a/examples/configs/opencode.provider.yml b/examples/configs/opencode.provider.yml index 10463849..8aabdd80 100644 --- a/examples/configs/opencode.provider.yml +++ b/examples/configs/opencode.provider.yml @@ -1,3 +1,20 @@ +# OpenCode Go prices ($/1M tokens) from https://opencode.ai/docs/go/ +# Qwen3.7 Plus / GPT 5.6 Luna use the lower (≤256K / ≤272K) tier. +# +# Model | input | output | cache_read | cache_write +# -------------------|--------|--------|------------|------------ +# Grok 4.5 | 2.00 | 6.00 | 0.30 | - +# Kimi K3 | 3.00 | 15.00 | 0.30 | - +# Qwen3.8 Max | 2.00 | 6.00 | 0.25 | 2.50 +# GLM-5.2 | 1.40 | 4.40 | 0.26 | - +# MiniMax M3 | 0.30 | 1.20 | 0.06 | - +# DeepSeek V4 Pro | 0.435 | 0.87 | 0.003625 | - +# GPT 5.6 Luna | 0.20 | 1.20 | 0.02 | 0.25 +# Qwen3.7 Plus | 0.40 | 1.60 | 0.04 | 0.50 +# Hy3 | 0.14 | 0.58 | 0.035 | - +# MiMo-V2.5 | 0.14 | 0.28 | 0.0028 | - +# DeepSeek V4 Flash | 0.14 | 0.28 | 0.0028 | - + simple: model: mimo-v2.5 temperature: 0.3 @@ -7,6 +24,10 @@ simple: extra_body: thinking: type: disabled + price: + input: 0.14 + output: 0.28 + cache_read: 0.0028 simple_json: model: mimo-v2.5 @@ -18,6 +39,10 @@ simple_json: extra_body: thinking: type: disabled + price: + input: 0.14 + output: 0.28 + cache_read: 0.0028 primary_agent: model: deepseek-v4-pro @@ -28,6 +53,10 @@ primary_agent: extra_body: thinking: type: enabled + price: + input: 0.435 + output: 0.87 + cache_read: 0.003625 assistant: model: deepseek-v4-pro @@ -38,6 +67,10 @@ assistant: extra_body: thinking: type: enabled + price: + input: 0.435 + output: 0.87 + cache_read: 0.003625 generator: model: deepseek-v4-pro @@ -48,6 +81,10 @@ generator: extra_body: thinking: type: enabled + price: + input: 0.435 + output: 0.87 + cache_read: 0.003625 refiner: model: deepseek-v4-pro @@ -58,9 +95,13 @@ refiner: extra_body: thinking: type: enabled + price: + input: 0.435 + output: 0.87 + cache_read: 0.003625 adviser: - model: deepseek-v4-pro + model: qwen3.7-plus n: 1 max_tokens: 16384 reasoning: @@ -68,6 +109,11 @@ adviser: extra_body: thinking: type: enabled + price: + input: 0.40 + output: 1.60 + cache_read: 0.04 + cache_write: 0.50 reflector: model: deepseek-v4-flash @@ -78,6 +124,10 @@ reflector: extra_body: thinking: type: disabled + price: + input: 0.14 + output: 0.28 + cache_read: 0.0028 searcher: model: deepseek-v4-flash @@ -88,6 +138,10 @@ searcher: extra_body: thinking: type: disabled + price: + input: 0.14 + output: 0.28 + cache_read: 0.0028 enricher: model: deepseek-v4-flash @@ -98,16 +152,24 @@ enricher: extra_body: thinking: type: disabled + price: + input: 0.14 + output: 0.28 + cache_read: 0.0028 coder: - model: deepseek-v4-pro + model: minimax-m3 n: 1 max_tokens: 20480 reasoning: effort: high extra_body: thinking: - type: enabled + type: adaptive + price: + input: 0.30 + output: 1.20 + cache_read: 0.06 installer: model: deepseek-v4-flash @@ -118,9 +180,13 @@ installer: extra_body: thinking: type: enabled + price: + input: 0.14 + output: 0.28 + cache_read: 0.0028 pentester: - model: deepseek-v4-pro + model: deepseek-v4-flash n: 1 max_tokens: 16384 reasoning: @@ -128,3 +194,7 @@ pentester: extra_body: thinking: type: enabled + price: + input: 0.14 + output: 0.28 + cache_read: 0.0028 diff --git a/examples/tests/opencode-report.md b/examples/tests/opencode-report.md index a061fb1e..02b1fe88 100644 --- a/examples/tests/opencode-report.md +++ b/examples/tests/opencode-report.md @@ -1,27 +1,27 @@ # LLM Agent Testing Report -Generated: Tue, 04 Aug 2026 17:52:20 UTC +Generated: Tue, 04 Aug 2026 20:10:25 UTC ## Overall Results | Agent | Model | Reasoning | Success Rate | Average Latency | |-------|-------|-----------|--------------|-----------------| -| simple | mimo-v2.5 | true | 24/24 (100.00%) | 3.686s | -| simple_json | mimo-v2.5 | false | 7/7 (100.00%) | 6.841s | -| primary_agent | deepseek-v4-pro | true | 24/24 (100.00%) | 2.749s | -| assistant | deepseek-v4-pro | true | 24/24 (100.00%) | 2.568s | -| generator | deepseek-v4-pro | true | 24/24 (100.00%) | 2.708s | -| refiner | deepseek-v4-pro | true | 24/24 (100.00%) | 2.575s | -| adviser | deepseek-v4-pro | true | 24/24 (100.00%) | 2.367s | -| reflector | deepseek-v4-flash | true | 24/24 (100.00%) | 1.641s | -| searcher | deepseek-v4-flash | true | 24/24 (100.00%) | 1.649s | -| enricher | deepseek-v4-flash | true | 24/24 (100.00%) | 1.620s | -| coder | deepseek-v4-pro | true | 24/24 (100.00%) | 2.590s | -| installer | deepseek-v4-flash | true | 24/24 (100.00%) | 1.822s | -| pentester | deepseek-v4-pro | true | 24/24 (100.00%) | 2.605s | +| simple | mimo-v2.5 | true | 24/24 (100.00%) | 3.713s | +| simple_json | mimo-v2.5 | false | 7/7 (100.00%) | 2.993s | +| primary_agent | deepseek-v4-pro | true | 24/24 (100.00%) | 2.459s | +| assistant | deepseek-v4-pro | true | 24/24 (100.00%) | 2.577s | +| generator | deepseek-v4-pro | true | 24/24 (100.00%) | 2.544s | +| refiner | deepseek-v4-pro | true | 24/24 (100.00%) | 2.571s | +| adviser | qwen3.7-plus | true | 24/24 (100.00%) | 6.186s | +| reflector | deepseek-v4-flash | true | 24/24 (100.00%) | 1.789s | +| searcher | deepseek-v4-flash | true | 24/24 (100.00%) | 1.782s | +| enricher | deepseek-v4-flash | true | 24/24 (100.00%) | 1.697s | +| coder | minimax-m3 | true | 23/24 (95.83%) | 1.892s | +| installer | deepseek-v4-flash | true | 24/24 (100.00%) | 1.906s | +| pentester | deepseek-v4-flash | true | 24/24 (100.00%) | 2.008s | -**Total**: 295/295 (100.00%) successful tests -**Overall average latency**: 2.487s +**Total**: 294/295 (99.66%) successful tests +**Overall average latency**: 2.603s ## Detailed Results @@ -31,39 +31,39 @@ Generated: Tue, 04 Aug 2026 17:52:20 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 5.883s | | -| Text Transform Uppercase | ✅ Pass | 4.343s | | -| Count from 1 to 5 | ✅ Pass | 4.122s | | -| Math Calculation | ✅ Pass | 3.864s | | -| Basic Echo Function | ✅ Pass | 3.019s | | -| Streaming Simple Math Streaming | ✅ Pass | 2.553s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 3.197s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 2.549s | | +| Text Transform Uppercase | ✅ Pass | 2.843s | | +| Simple Math | ✅ Pass | 7.262s | | +| Count from 1 to 5 | ✅ Pass | 2.157s | | +| Math Calculation | ✅ Pass | 1.826s | | +| Basic Echo Function | ✅ Pass | 1.861s | | +| Streaming Simple Math Streaming | ✅ Pass | 3.729s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.266s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 2.107s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 2.365s | | -| Search Query Function | ✅ Pass | 2.015s | | -| Ask Advice Function | ✅ Pass | 2.025s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.728s | | -| Basic Context Memory Test | ✅ Pass | 2.968s | | -| Function Argument Memory Test | ✅ Pass | 1.561s | | -| Function Response Memory Test | ✅ Pass | 1.278s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.662s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 2.393s | | -| Read a file, then edit it via unified diff | ✅ Pass | 4.179s | | -| Penetration Testing Methodology | ✅ Pass | 5.625s | | -| Vulnerability Assessment Tools | ✅ Pass | 9.226s | | -| SQL Injection Attack Type | ✅ Pass | 7.439s | | -| Penetration Testing Framework | ✅ Pass | 4.882s | | -| Web Application Security Scanner | ✅ Pass | 3.887s | | -| Penetration Testing Tool Selection | ✅ Pass | 4.694s | | +| JSON Response Function | ✅ Pass | 1.934s | | +| Search Query Function | ✅ Pass | 1.953s | | +| Ask Advice Function | ✅ Pass | 5.166s | | +| Streaming Search Query Function Streaming | ✅ Pass | 7.402s | | +| Basic Context Memory Test | ✅ Pass | 4.311s | | +| Function Argument Memory Test | ✅ Pass | 1.958s | | +| Function Response Memory Test | ✅ Pass | 1.940s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 3.350s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 4.064s | | +| Read a file, then edit it via unified diff | ✅ Pass | 8.963s | | +| Penetration Testing Methodology | ✅ Pass | 4.230s | | +| Vulnerability Assessment Tools | ✅ Pass | 6.164s | | +| SQL Injection Attack Type | ✅ Pass | 4.165s | | +| Penetration Testing Framework | ✅ Pass | 3.052s | | +| Web Application Security Scanner | ✅ Pass | 3.987s | | +| Penetration Testing Tool Selection | ✅ Pass | 3.404s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 3.686s +**Average latency**: 3.713s --- @@ -73,22 +73,22 @@ Generated: Tue, 04 Aug 2026 17:52:20 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Vulnerability Report Memory Test | ✅ Pass | 5.392s | | -| Person Information JSON | ✅ Pass | 1.719s | | -| Project Information JSON | ✅ Pass | 1.303s | | -| User Profile JSON | ✅ Pass | 2.431s | | -| Streaming Person Information JSON Streaming | ✅ Pass | 1.374s | | -| JSON Array Response Without Schema | ✅ Pass | 32.140s | | +| Vulnerability Report Memory Test | ✅ Pass | 4.877s | | +| Person Information JSON | ✅ Pass | 2.944s | | +| Project Information JSON | ✅ Pass | 2.369s | | +| User Profile JSON | ✅ Pass | 3.358s | | +| JSON Array Response Without Schema | ✅ Pass | 3.220s | | +| Streaming Person Information JSON Streaming | ✅ Pass | 3.122s | | #### Capability Tests | Test | Capability | Result | Latency | Note | |------|------------|--------|---------|------| -| Structured Output With JSON Schema | structured_output | ✅ Pass | 3.525s | | +| Structured Output With JSON Schema | structured_output | ✅ Pass | 1.060s | | **Summary**: 7/7 (100.00%) successful tests -**Average latency**: 6.841s +**Average latency**: 2.993s --- @@ -98,39 +98,39 @@ Generated: Tue, 04 Aug 2026 17:52:20 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 1.795s | | -| Text Transform Uppercase | ✅ Pass | 1.751s | | -| Count from 1 to 5 | ✅ Pass | 1.684s | | -| Math Calculation | ✅ Pass | 1.966s | | -| Basic Echo Function | ✅ Pass | 1.984s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.624s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.726s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 2.163s | | +| Simple Math | ✅ Pass | 2.190s | | +| Text Transform Uppercase | ✅ Pass | 1.829s | | +| Count from 1 to 5 | ✅ Pass | 1.928s | | +| Math Calculation | ✅ Pass | 1.894s | | +| Basic Echo Function | ✅ Pass | 1.851s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.585s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.695s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 2.054s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 2.286s | | -| Search Query Function | ✅ Pass | 2.207s | | -| Ask Advice Function | ✅ Pass | 2.326s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.885s | | -| Basic Context Memory Test | ✅ Pass | 2.461s | | -| Function Argument Memory Test | ✅ Pass | 1.767s | | -| Function Response Memory Test | ✅ Pass | 2.339s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 3.399s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 2.215s | | -| Read a file, then edit it via unified diff | ✅ Pass | 5.852s | | -| Penetration Testing Methodology | ✅ Pass | 3.285s | | -| Vulnerability Assessment Tools | ✅ Pass | 9.214s | | -| SQL Injection Attack Type | ✅ Pass | 3.741s | | -| Penetration Testing Framework | ✅ Pass | 3.013s | | -| Web Application Security Scanner | ✅ Pass | 2.833s | | -| Penetration Testing Tool Selection | ✅ Pass | 2.446s | | +| JSON Response Function | ✅ Pass | 2.716s | | +| Search Query Function | ✅ Pass | 2.367s | | +| Ask Advice Function | ✅ Pass | 2.249s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.819s | | +| Basic Context Memory Test | ✅ Pass | 2.530s | | +| Function Argument Memory Test | ✅ Pass | 1.934s | | +| Function Response Memory Test | ✅ Pass | 2.199s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.946s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 2.406s | | +| Read a file, then edit it via unified diff | ✅ Pass | 5.338s | | +| Penetration Testing Methodology | ✅ Pass | 3.498s | | +| Vulnerability Assessment Tools | ✅ Pass | 4.697s | | +| SQL Injection Attack Type | ✅ Pass | 2.377s | | +| Penetration Testing Framework | ✅ Pass | 2.428s | | +| Web Application Security Scanner | ✅ Pass | 2.540s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.924s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 2.749s +**Average latency**: 2.459s --- @@ -140,39 +140,39 @@ Generated: Tue, 04 Aug 2026 17:52:20 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 2.218s | | -| Text Transform Uppercase | ✅ Pass | 1.915s | | -| Count from 1 to 5 | ✅ Pass | 1.788s | | -| Math Calculation | ✅ Pass | 1.848s | | -| Basic Echo Function | ✅ Pass | 1.976s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.601s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.786s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 2.336s | | +| Simple Math | ✅ Pass | 2.139s | | +| Text Transform Uppercase | ✅ Pass | 2.014s | | +| Count from 1 to 5 | ✅ Pass | 1.832s | | +| Math Calculation | ✅ Pass | 1.560s | | +| Basic Echo Function | ✅ Pass | 1.986s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.981s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.861s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 2.498s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 2.348s | | -| Search Query Function | ✅ Pass | 2.075s | | -| Ask Advice Function | ✅ Pass | 2.073s | | -| Streaming Search Query Function Streaming | ✅ Pass | 2.031s | | -| Basic Context Memory Test | ✅ Pass | 2.063s | | -| Function Argument Memory Test | ✅ Pass | 2.068s | | -| Function Response Memory Test | ✅ Pass | 1.961s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.920s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 2.182s | | -| Read a file, then edit it via unified diff | ✅ Pass | 5.868s | | -| Penetration Testing Methodology | ✅ Pass | 3.612s | | -| Vulnerability Assessment Tools | ✅ Pass | 6.697s | | -| SQL Injection Attack Type | ✅ Pass | 2.484s | | -| Penetration Testing Framework | ✅ Pass | 3.598s | | -| Web Application Security Scanner | ✅ Pass | 1.871s | | -| Penetration Testing Tool Selection | ✅ Pass | 2.303s | | +| JSON Response Function | ✅ Pass | 2.482s | | +| Search Query Function | ✅ Pass | 2.020s | | +| Ask Advice Function | ✅ Pass | 2.770s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.853s | | +| Basic Context Memory Test | ✅ Pass | 2.040s | | +| Function Argument Memory Test | ✅ Pass | 2.032s | | +| Function Response Memory Test | ✅ Pass | 2.106s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.957s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 2.265s | | +| Read a file, then edit it via unified diff | ✅ Pass | 5.237s | | +| Penetration Testing Methodology | ✅ Pass | 3.523s | | +| Vulnerability Assessment Tools | ✅ Pass | 5.750s | | +| SQL Injection Attack Type | ✅ Pass | 3.153s | | +| Penetration Testing Framework | ✅ Pass | 3.118s | | +| Web Application Security Scanner | ✅ Pass | 2.198s | | +| Penetration Testing Tool Selection | ✅ Pass | 2.471s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 2.568s +**Average latency**: 2.577s --- @@ -182,39 +182,39 @@ Generated: Tue, 04 Aug 2026 17:52:20 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 2.051s | | -| Text Transform Uppercase | ✅ Pass | 2.001s | | -| Count from 1 to 5 | ✅ Pass | 1.791s | | -| Math Calculation | ✅ Pass | 1.544s | | -| Basic Echo Function | ✅ Pass | 1.966s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.640s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.907s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 2.238s | | +| Simple Math | ✅ Pass | 2.088s | | +| Text Transform Uppercase | ✅ Pass | 1.740s | | +| Count from 1 to 5 | ✅ Pass | 2.220s | | +| Math Calculation | ✅ Pass | 1.708s | | +| Basic Echo Function | ✅ Pass | 1.918s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.807s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.792s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.970s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 2.337s | | -| Search Query Function | ✅ Pass | 2.173s | | -| Ask Advice Function | ✅ Pass | 2.341s | | -| Streaming Search Query Function Streaming | ✅ Pass | 2.026s | | -| Basic Context Memory Test | ✅ Pass | 2.881s | | -| Function Argument Memory Test | ✅ Pass | 2.004s | | -| Function Response Memory Test | ✅ Pass | 1.953s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 3.369s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 1.852s | | -| Read a file, then edit it via unified diff | ✅ Pass | 6.872s | | -| Penetration Testing Methodology | ✅ Pass | 3.517s | | -| Vulnerability Assessment Tools | ✅ Pass | 7.270s | | -| SQL Injection Attack Type | ✅ Pass | 2.615s | | -| Penetration Testing Framework | ✅ Pass | 3.764s | | -| Web Application Security Scanner | ✅ Pass | 2.607s | | -| Penetration Testing Tool Selection | ✅ Pass | 2.266s | | +| JSON Response Function | ✅ Pass | 1.998s | | +| Search Query Function | ✅ Pass | 1.964s | | +| Ask Advice Function | ✅ Pass | 2.670s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.780s | | +| Basic Context Memory Test | ✅ Pass | 3.211s | | +| Function Argument Memory Test | ✅ Pass | 2.128s | | +| Function Response Memory Test | ✅ Pass | 2.162s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.919s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.929s | | +| Read a file, then edit it via unified diff | ✅ Pass | 4.732s | | +| Penetration Testing Methodology | ✅ Pass | 3.777s | | +| Vulnerability Assessment Tools | ✅ Pass | 6.549s | | +| SQL Injection Attack Type | ✅ Pass | 2.130s | | +| Penetration Testing Framework | ✅ Pass | 2.684s | | +| Web Application Security Scanner | ✅ Pass | 2.727s | | +| Penetration Testing Tool Selection | ✅ Pass | 2.440s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 2.708s +**Average latency**: 2.544s --- @@ -224,81 +224,81 @@ Generated: Tue, 04 Aug 2026 17:52:20 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 2.014s | | -| Text Transform Uppercase | ✅ Pass | 1.849s | | -| Count from 1 to 5 | ✅ Pass | 1.946s | | -| Math Calculation | ✅ Pass | 1.772s | | -| Basic Echo Function | ✅ Pass | 1.842s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.569s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.883s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 2.039s | | +| Simple Math | ✅ Pass | 2.062s | | +| Text Transform Uppercase | ✅ Pass | 1.854s | | +| Count from 1 to 5 | ✅ Pass | 1.989s | | +| Math Calculation | ✅ Pass | 1.321s | | +| Basic Echo Function | ✅ Pass | 1.748s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.654s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.947s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.812s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 2.522s | | -| Search Query Function | ✅ Pass | 2.253s | | -| Ask Advice Function | ✅ Pass | 2.164s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.933s | | -| Basic Context Memory Test | ✅ Pass | 2.625s | | -| Function Argument Memory Test | ✅ Pass | 2.138s | | -| Function Response Memory Test | ✅ Pass | 2.165s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 3.266s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 2.034s | | -| Read a file, then edit it via unified diff | ✅ Pass | 5.432s | | -| Penetration Testing Methodology | ✅ Pass | 3.023s | | -| Vulnerability Assessment Tools | ✅ Pass | 7.027s | | -| SQL Injection Attack Type | ✅ Pass | 1.861s | | -| Penetration Testing Framework | ✅ Pass | 3.503s | | -| Web Application Security Scanner | ✅ Pass | 2.864s | | -| Penetration Testing Tool Selection | ✅ Pass | 2.055s | | +| JSON Response Function | ✅ Pass | 2.266s | | +| Search Query Function | ✅ Pass | 1.919s | | +| Ask Advice Function | ✅ Pass | 2.668s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.946s | | +| Basic Context Memory Test | ✅ Pass | 2.427s | | +| Function Argument Memory Test | ✅ Pass | 1.932s | | +| Function Response Memory Test | ✅ Pass | 1.962s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 3.524s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.940s | | +| Read a file, then edit it via unified diff | ✅ Pass | 5.215s | | +| Penetration Testing Methodology | ✅ Pass | 3.689s | | +| Vulnerability Assessment Tools | ✅ Pass | 6.487s | | +| SQL Injection Attack Type | ✅ Pass | 3.090s | | +| Penetration Testing Framework | ✅ Pass | 3.394s | | +| Web Application Security Scanner | ✅ Pass | 2.363s | | +| Penetration Testing Tool Selection | ✅ Pass | 2.492s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 2.575s +**Average latency**: 2.571s --- -### adviser (deepseek-v4-pro) +### adviser (qwen3.7-plus) #### Basic Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 2.105s | | -| Text Transform Uppercase | ✅ Pass | 1.905s | | -| Count from 1 to 5 | ✅ Pass | 1.790s | | -| Math Calculation | ✅ Pass | 1.711s | | -| Basic Echo Function | ✅ Pass | 1.962s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.581s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.914s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 2.500s | | +| Simple Math | ✅ Pass | 3.453s | | +| Text Transform Uppercase | ✅ Pass | 6.036s | | +| Count from 1 to 5 | ✅ Pass | 5.461s | | +| Math Calculation | ✅ Pass | 4.543s | | +| Basic Echo Function | ✅ Pass | 5.292s | | +| Streaming Simple Math Streaming | ✅ Pass | 3.027s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 4.731s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 8.427s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 2.423s | | -| Search Query Function | ✅ Pass | 1.798s | | -| Ask Advice Function | ✅ Pass | 2.050s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.925s | | -| Basic Context Memory Test | ✅ Pass | 2.906s | | -| Function Argument Memory Test | ✅ Pass | 1.827s | | -| Function Response Memory Test | ✅ Pass | 2.000s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.765s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 1.994s | | -| Read a file, then edit it via unified diff | ✅ Pass | 5.798s | | -| Penetration Testing Methodology | ✅ Pass | 3.265s | | -| Vulnerability Assessment Tools | ✅ Pass | 2.125s | | -| SQL Injection Attack Type | ✅ Pass | 1.931s | | -| Penetration Testing Framework | ✅ Pass | 3.723s | | -| Web Application Security Scanner | ✅ Pass | 2.619s | | -| Penetration Testing Tool Selection | ✅ Pass | 2.168s | | +| JSON Response Function | ✅ Pass | 2.548s | | +| Ask Advice Function | ✅ Pass | 6.060s | | +| Streaming Search Query Function Streaming | ✅ Pass | 4.008s | | +| Basic Context Memory Test | ✅ Pass | 4.596s | | +| Search Query Function | ✅ Pass | 16.296s | | +| Function Argument Memory Test | ✅ Pass | 4.500s | | +| Function Response Memory Test | ✅ Pass | 5.568s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 4.007s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 9.534s | | +| Read a file, then edit it via unified diff | ✅ Pass | 5.928s | | +| Penetration Testing Methodology | ✅ Pass | 7.995s | | +| Vulnerability Assessment Tools | ✅ Pass | 11.838s | | +| SQL Injection Attack Type | ✅ Pass | 9.801s | | +| Penetration Testing Framework | ✅ Pass | 7.005s | | +| Web Application Security Scanner | ✅ Pass | 5.210s | | +| Penetration Testing Tool Selection | ✅ Pass | 2.585s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 2.367s +**Average latency**: 6.186s --- @@ -308,39 +308,39 @@ Generated: Tue, 04 Aug 2026 17:52:20 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 1.504s | | -| Text Transform Uppercase | ✅ Pass | 1.061s | | -| Count from 1 to 5 | ✅ Pass | 1.305s | | -| Math Calculation | ✅ Pass | 1.039s | | -| Basic Echo Function | ✅ Pass | 1.373s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.329s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.145s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 1.904s | | +| Simple Math | ✅ Pass | 1.728s | | +| Text Transform Uppercase | ✅ Pass | 3.622s | | +| Count from 1 to 5 | ✅ Pass | 1.807s | | +| Math Calculation | ✅ Pass | 1.388s | | +| Basic Echo Function | ✅ Pass | 1.791s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.440s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.345s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.886s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 1.892s | | -| Search Query Function | ✅ Pass | 1.410s | | -| Ask Advice Function | ✅ Pass | 1.482s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.395s | | -| Basic Context Memory Test | ✅ Pass | 2.046s | | -| Function Argument Memory Test | ✅ Pass | 1.326s | | -| Function Response Memory Test | ✅ Pass | 1.188s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.115s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 1.240s | | -| Read a file, then edit it via unified diff | ✅ Pass | 3.529s | | -| Penetration Testing Methodology | ✅ Pass | 2.082s | | -| Vulnerability Assessment Tools | ✅ Pass | 3.821s | | -| SQL Injection Attack Type | ✅ Pass | 1.225s | | -| Penetration Testing Framework | ✅ Pass | 1.196s | | -| Web Application Security Scanner | ✅ Pass | 1.281s | | -| Penetration Testing Tool Selection | ✅ Pass | 1.485s | | +| JSON Response Function | ✅ Pass | 1.424s | | +| Search Query Function | ✅ Pass | 1.430s | | +| Ask Advice Function | ✅ Pass | 1.501s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.633s | | +| Basic Context Memory Test | ✅ Pass | 1.176s | | +| Function Argument Memory Test | ✅ Pass | 1.317s | | +| Function Response Memory Test | ✅ Pass | 1.435s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.016s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.374s | | +| Read a file, then edit it via unified diff | ✅ Pass | 4.120s | | +| Penetration Testing Methodology | ✅ Pass | 1.528s | | +| Vulnerability Assessment Tools | ✅ Pass | 3.710s | | +| SQL Injection Attack Type | ✅ Pass | 1.378s | | +| Penetration Testing Framework | ✅ Pass | 1.265s | | +| Web Application Security Scanner | ✅ Pass | 1.144s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.473s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 1.641s +**Average latency**: 1.789s --- @@ -350,39 +350,39 @@ Generated: Tue, 04 Aug 2026 17:52:20 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 1.730s | | -| Text Transform Uppercase | ✅ Pass | 1.125s | | -| Count from 1 to 5 | ✅ Pass | 1.067s | | -| Math Calculation | ✅ Pass | 1.210s | | -| Basic Echo Function | ✅ Pass | 1.649s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.177s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.379s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 1.614s | | +| Simple Math | ✅ Pass | 2.503s | | +| Text Transform Uppercase | ✅ Pass | 3.224s | | +| Count from 1 to 5 | ✅ Pass | 1.959s | | +| Math Calculation | ✅ Pass | 1.419s | | +| Basic Echo Function | ✅ Pass | 1.743s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.493s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.151s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.544s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 1.704s | | -| Search Query Function | ✅ Pass | 1.836s | | -| Ask Advice Function | ✅ Pass | 1.652s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.415s | | -| Basic Context Memory Test | ✅ Pass | 1.638s | | -| Function Argument Memory Test | ✅ Pass | 1.380s | | -| Function Response Memory Test | ✅ Pass | 1.164s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 1.771s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 1.303s | | -| Read a file, then edit it via unified diff | ✅ Pass | 4.305s | | -| Penetration Testing Methodology | ✅ Pass | 1.201s | | -| Vulnerability Assessment Tools | ✅ Pass | 3.574s | | -| SQL Injection Attack Type | ✅ Pass | 1.253s | | -| Penetration Testing Framework | ✅ Pass | 1.467s | | -| Web Application Security Scanner | ✅ Pass | 1.408s | | -| Penetration Testing Tool Selection | ✅ Pass | 1.542s | | +| JSON Response Function | ✅ Pass | 1.301s | | +| Search Query Function | ✅ Pass | 1.534s | | +| Ask Advice Function | ✅ Pass | 1.379s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.606s | | +| Basic Context Memory Test | ✅ Pass | 1.167s | | +| Function Argument Memory Test | ✅ Pass | 1.551s | | +| Function Response Memory Test | ✅ Pass | 1.303s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.799s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.664s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.595s | | +| Penetration Testing Methodology | ✅ Pass | 1.626s | | +| Vulnerability Assessment Tools | ✅ Pass | 4.019s | | +| SQL Injection Attack Type | ✅ Pass | 1.457s | | +| Penetration Testing Framework | ✅ Pass | 1.149s | | +| Web Application Security Scanner | ✅ Pass | 1.164s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.409s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 1.649s +**Average latency**: 1.782s --- @@ -392,81 +392,81 @@ Generated: Tue, 04 Aug 2026 17:52:20 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 1.181s | | -| Text Transform Uppercase | ✅ Pass | 1.918s | | -| Count from 1 to 5 | ✅ Pass | 1.154s | | -| Math Calculation | ✅ Pass | 1.134s | | -| Basic Echo Function | ✅ Pass | 1.340s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.069s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.501s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 1.747s | | +| Simple Math | ✅ Pass | 1.617s | | +| Text Transform Uppercase | ✅ Pass | 2.610s | | +| Count from 1 to 5 | ✅ Pass | 1.770s | | +| Math Calculation | ✅ Pass | 1.612s | | +| Basic Echo Function | ✅ Pass | 1.670s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.229s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.867s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.760s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 1.846s | | -| Search Query Function | ✅ Pass | 1.649s | | -| Ask Advice Function | ✅ Pass | 1.378s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.847s | | -| Basic Context Memory Test | ✅ Pass | 1.439s | | -| Function Argument Memory Test | ✅ Pass | 1.299s | | -| Function Response Memory Test | ✅ Pass | 1.073s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.009s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 1.379s | | -| Read a file, then edit it via unified diff | ✅ Pass | 3.807s | | -| Penetration Testing Methodology | ✅ Pass | 1.163s | | -| Vulnerability Assessment Tools | ✅ Pass | 4.037s | | -| SQL Injection Attack Type | ✅ Pass | 1.165s | | -| Penetration Testing Framework | ✅ Pass | 1.171s | | -| Web Application Security Scanner | ✅ Pass | 1.101s | | -| Penetration Testing Tool Selection | ✅ Pass | 1.451s | | +| JSON Response Function | ✅ Pass | 1.479s | | +| Search Query Function | ✅ Pass | 1.527s | | +| Ask Advice Function | ✅ Pass | 1.515s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.670s | | +| Basic Context Memory Test | ✅ Pass | 1.797s | | +| Function Argument Memory Test | ✅ Pass | 1.206s | | +| Function Response Memory Test | ✅ Pass | 1.115s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.794s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.449s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.797s | | +| Penetration Testing Methodology | ✅ Pass | 1.360s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.952s | | +| SQL Injection Attack Type | ✅ Pass | 1.278s | | +| Penetration Testing Framework | ✅ Pass | 1.183s | | +| Web Application Security Scanner | ✅ Pass | 1.097s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.365s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 1.620s +**Average latency**: 1.697s --- -### coder (deepseek-v4-pro) +### coder (minimax-m3) #### Basic Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 1.750s | | -| Text Transform Uppercase | ✅ Pass | 1.944s | | -| Count from 1 to 5 | ✅ Pass | 1.841s | | -| Math Calculation | ✅ Pass | 1.683s | | -| Basic Echo Function | ✅ Pass | 2.016s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.628s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 2.298s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 2.437s | | +| Simple Math | ✅ Pass | 1.134s | | +| Text Transform Uppercase | ✅ Pass | 1.002s | | +| Count from 1 to 5 | ✅ Pass | 1.186s | | +| Math Calculation | ✅ Pass | 1.072s | | +| Basic Echo Function | ✅ Pass | 1.217s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.034s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.733s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 2.039s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 2.144s | | -| Search Query Function | ✅ Pass | 2.338s | | -| Ask Advice Function | ✅ Pass | 2.100s | | -| Streaming Search Query Function Streaming | ✅ Pass | 2.352s | | -| Basic Context Memory Test | ✅ Pass | 2.277s | | -| Function Argument Memory Test | ✅ Pass | 2.404s | | -| Function Response Memory Test | ✅ Pass | 2.149s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 3.142s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 2.159s | | -| Read a file, then edit it via unified diff | ✅ Pass | 5.470s | | -| Penetration Testing Methodology | ✅ Pass | 3.792s | | -| Vulnerability Assessment Tools | ✅ Pass | 6.231s | | -| SQL Injection Attack Type | ✅ Pass | 1.975s | | -| Penetration Testing Framework | ✅ Pass | 3.089s | | -| Web Application Security Scanner | ✅ Pass | 2.527s | | -| Penetration Testing Tool Selection | ✅ Pass | 2.395s | | +| JSON Response Function | ✅ Pass | 1.153s | | +| Search Query Function | ✅ Pass | 1.322s | | +| Ask Advice Function | ✅ Pass | 1.559s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.922s | | +| Basic Context Memory Test | ✅ Pass | 1.357s | | +| Function Argument Memory Test | ✅ Pass | 1.183s | | +| Function Response Memory Test | ✅ Pass | 1.159s | | +| Penetration Testing Memory with Tool Call | ❌ Fail | 1.749s | expected function 'generate\_report' not found in tool calls: expected function generate\_report not found in tool calls | +| Cybersecurity Workflow Memory Test | ✅ Pass | 0.982s | | +| Read a file, then edit it via unified diff | ✅ Pass | 5.263s | | +| Penetration Testing Methodology | ✅ Pass | 4.101s | | +| Vulnerability Assessment Tools | ✅ Pass | 3.509s | | +| SQL Injection Attack Type | ✅ Pass | 1.767s | | +| Penetration Testing Framework | ✅ Pass | 4.416s | | +| Web Application Security Scanner | ✅ Pass | 2.111s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.437s | | -**Summary**: 24/24 (100.00%) successful tests +**Summary**: 23/24 (95.83%) successful tests -**Average latency**: 2.590s +**Average latency**: 1.892s --- @@ -476,81 +476,81 @@ Generated: Tue, 04 Aug 2026 17:52:20 UTC | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 1.266s | | -| Text Transform Uppercase | ✅ Pass | 1.353s | | -| Count from 1 to 5 | ✅ Pass | 1.448s | | -| Math Calculation | ✅ Pass | 1.556s | | -| Basic Echo Function | ✅ Pass | 1.578s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.299s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.994s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 1.900s | | +| Simple Math | ✅ Pass | 1.994s | | +| Text Transform Uppercase | ✅ Pass | 2.642s | | +| Count from 1 to 5 | ✅ Pass | 1.896s | | +| Math Calculation | ✅ Pass | 2.042s | | +| Basic Echo Function | ✅ Pass | 1.909s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.611s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 2.163s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.951s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 1.531s | | -| Search Query Function | ✅ Pass | 1.864s | | -| Ask Advice Function | ✅ Pass | 1.934s | | -| Streaming Search Query Function Streaming | ✅ Pass | 1.659s | | -| Basic Context Memory Test | ✅ Pass | 2.764s | | -| Function Argument Memory Test | ✅ Pass | 1.408s | | -| Function Response Memory Test | ✅ Pass | 1.847s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 2.287s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 1.778s | | -| Read a file, then edit it via unified diff | ✅ Pass | 3.633s | | -| Penetration Testing Methodology | ✅ Pass | 1.422s | | -| Vulnerability Assessment Tools | ✅ Pass | 2.364s | | -| SQL Injection Attack Type | ✅ Pass | 1.802s | | -| Penetration Testing Framework | ✅ Pass | 1.336s | | -| Web Application Security Scanner | ✅ Pass | 1.525s | | -| Penetration Testing Tool Selection | ✅ Pass | 2.178s | | +| JSON Response Function | ✅ Pass | 1.582s | | +| Search Query Function | ✅ Pass | 1.525s | | +| Ask Advice Function | ✅ Pass | 1.512s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.641s | | +| Basic Context Memory Test | ✅ Pass | 2.317s | | +| Function Argument Memory Test | ✅ Pass | 1.634s | | +| Function Response Memory Test | ✅ Pass | 1.345s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 2.281s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 1.808s | | +| Read a file, then edit it via unified diff | ✅ Pass | 4.274s | | +| Penetration Testing Methodology | ✅ Pass | 1.342s | | +| Vulnerability Assessment Tools | ✅ Pass | 2.158s | | +| SQL Injection Attack Type | ✅ Pass | 1.707s | | +| Penetration Testing Framework | ✅ Pass | 1.398s | | +| Web Application Security Scanner | ✅ Pass | 1.314s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.680s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 1.822s +**Average latency**: 1.906s --- -### pentester (deepseek-v4-pro) +### pentester (deepseek-v4-flash) #### Basic Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| Simple Math | ✅ Pass | 2.151s | | -| Text Transform Uppercase | ✅ Pass | 1.506s | | -| Count from 1 to 5 | ✅ Pass | 1.940s | | -| Math Calculation | ✅ Pass | 1.607s | | -| Basic Echo Function | ✅ Pass | 2.126s | | -| Streaming Simple Math Streaming | ✅ Pass | 1.636s | | -| Streaming Count from 1 to 3 Streaming | ✅ Pass | 3.010s | | -| Streaming Basic Echo Function Streaming | ✅ Pass | 2.324s | | +| Simple Math | ✅ Pass | 2.183s | | +| Text Transform Uppercase | ✅ Pass | 3.149s | | +| Count from 1 to 5 | ✅ Pass | 1.765s | | +| Math Calculation | ✅ Pass | 1.944s | | +| Basic Echo Function | ✅ Pass | 1.904s | | +| Streaming Simple Math Streaming | ✅ Pass | 1.358s | | +| Streaming Count from 1 to 3 Streaming | ✅ Pass | 1.945s | | +| Streaming Basic Echo Function Streaming | ✅ Pass | 1.824s | | #### Advanced Tests | Test | Result | Latency | Error | |------|--------|---------|-------| -| JSON Response Function | ✅ Pass | 2.441s | | -| Search Query Function | ✅ Pass | 2.244s | | -| Ask Advice Function | ✅ Pass | 2.284s | | -| Streaming Search Query Function Streaming | ✅ Pass | 2.421s | | -| Basic Context Memory Test | ✅ Pass | 1.935s | | -| Function Argument Memory Test | ✅ Pass | 2.131s | | -| Function Response Memory Test | ✅ Pass | 2.362s | | -| Penetration Testing Memory with Tool Call | ✅ Pass | 3.331s | | -| Cybersecurity Workflow Memory Test | ✅ Pass | 1.897s | | -| Read a file, then edit it via unified diff | ✅ Pass | 5.897s | | -| Penetration Testing Methodology | ✅ Pass | 4.382s | | -| Vulnerability Assessment Tools | ✅ Pass | 3.823s | | -| SQL Injection Attack Type | ✅ Pass | 2.507s | | -| Penetration Testing Framework | ✅ Pass | 4.464s | | -| Web Application Security Scanner | ✅ Pass | 1.676s | | -| Penetration Testing Tool Selection | ✅ Pass | 2.410s | | +| JSON Response Function | ✅ Pass | 1.577s | | +| Search Query Function | ✅ Pass | 1.392s | | +| Ask Advice Function | ✅ Pass | 1.713s | | +| Streaming Search Query Function Streaming | ✅ Pass | 1.555s | | +| Basic Context Memory Test | ✅ Pass | 2.392s | | +| Function Argument Memory Test | ✅ Pass | 1.696s | | +| Function Response Memory Test | ✅ Pass | 1.686s | | +| Penetration Testing Memory with Tool Call | ✅ Pass | 1.942s | | +| Cybersecurity Workflow Memory Test | ✅ Pass | 2.330s | | +| Read a file, then edit it via unified diff | ✅ Pass | 3.631s | | +| Penetration Testing Methodology | ✅ Pass | 1.553s | | +| Vulnerability Assessment Tools | ✅ Pass | 3.893s | | +| SQL Injection Attack Type | ✅ Pass | 2.072s | | +| Penetration Testing Framework | ✅ Pass | 1.467s | | +| Web Application Security Scanner | ✅ Pass | 1.436s | | +| Penetration Testing Tool Selection | ✅ Pass | 1.772s | | **Summary**: 24/24 (100.00%) successful tests -**Average latency**: 2.605s +**Average latency**: 2.008s ---