diff --git a/README.md b/README.md index e49c7d01..06d41c81 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ # mcpmark-release - Evaluation Results -Generated: 2025-09-18T12:47:26.868861 - -💸 Total Cost (so far): $12,417.45 +Generated: 2025-11-26T15:49:34.853971 +Task set: standard ## Overall Performance @@ -11,21 +10,25 @@ Generated: 2025-09-18T12:47:26.868861 | gpt-5-medium | 127 | 52.6% ± 1.3% | 68.5% | 33.9% | $127.46 | 478.2 | | gpt-5-high | 127 | 51.6% ± 2.5% | 66.1% | 33.1% | $153.89 | 1029.3 | | gpt-5-low | 127 | 46.9% ± 2.9% | 63.0% | 26.8% | $125.87 | 385.8 | +| claude-sonnet-4-5 | 127 | 32.1% ± 2.3% | 46.5% | 16.5% | / | 173.2 | | grok-4 | 127 | 31.7% ± 2.9% | 44.9% | 18.1% | $257.41 | 319.8 | | gpt-5-mini-high | 127 | 30.3% ± 1.7% | 46.5% | 16.5% | $40.35 | 349.9 | -| claude-opus-4-1 | 127 | 29.9% ± 0.0% | / | / | $1165.45 | 361.8 | | claude-sonnet-4-high | 127 | 28.3% ± 2.4% | 40.9% | 18.1% | $442.33 | 185.6 | | claude-sonnet-4 | 127 | 28.1% ± 2.6% | 44.9% | 12.6% | $252.41 | 218.3 | | claude-sonnet-4-low | 127 | 27.4% ± 1.7% | 39.4% | 18.1% | $460.95 | 199.4 | | gpt-5-mini-medium | 127 | 27.4% ± 3.1% | 45.7% | 9.4% | $26.02 | 159.9 | | o3 | 127 | 25.4% ± 2.0% | 43.3% | 12.6% | $113.94 | 169.4 | | qwen-3-coder-plus | 127 | 24.8% ± 2.1% | 40.9% | 12.6% | $36.46 | 274.3 | +| grok-4-fast | 127 | 24.0% ± 3.1% | 38.6% | 12.6% | / | 109.9 | +| glm-4-6 | 127 | 23.6% ± 1.5% | 36.2% | 13.4% | / | 243.0 | | kimi-k2-0905 | 127 | 21.9% ± 1.2% | 31.5% | 12.6% | $72.57 | 493.8 | +| deepseek-v3-1-terminus-thinking | 127 | 21.3% ± 3.3% | 37.0% | 5.5% | / | 734.5 | | grok-code-fast-1 | 127 | 20.5% ± 3.4% | 30.7% | 9.4% | $16.08 | 156.6 | | kimi-k2-0711 | 127 | 19.1% ± 1.6% | 31.5% | 11.8% | $36.45 | 214.8 | | qwen-3-max | 127 | 17.7% ± 1.3% | 22.8% | 11.0% | $160 | 213.6 | | o4-mini | 127 | 17.3% ± 2.3% | 31.5% | 6.3% | $63.62 | 323.3 | | deepseek-chat | 127 | 16.7% ± 1.4% | 28.3% | 7.9% | $35.66 | 269.9 | +| deepseek-v3-1-terminus | 127 | 16.5% ± 5.1% | 29.9% | 3.9% | / | 244.9 | | gemini-2-5-pro | 127 | 15.8% ± 0.6% | 29.9% | 4.7% | $162.48 | 119.4 | | glm-4-5 | 127 | 15.6% ± 1.2% | 24.4% | 6.3% | $18.27 | 166.3 | | gemini-2-5-flash | 127 | 9.1% ± 0.7% | 18.1% | 3.9% | $41.81 | 114.9 | @@ -48,10 +51,12 @@ Generated: 2025-09-18T12:47:26.868861 | grok-4 | 30 | 50.8% ± 6.4% | 73.3% | 26.7% | $27.08 | 256.6 | | o3 | 30 | 35.8% ± 2.8% | 50.0% | 26.7% | $45.65 | 277.9 | | gpt-5-mini-high | 30 | 35.0% ± 7.6% | 46.7% | 23.3% | $5.56 | 288.9 | -| claude-opus-4-1 | 30 | 33.3% ± 0.0% | / | / | $132.3 | 267.8 | | gpt-5-mini-medium | 30 | 33.3% ± 6.2% | 53.3% | 10.0% | $3.74 | 174.3 | +| claude-sonnet-4-5 | 30 | 32.5% ± 4.9% | 43.3% | 13.3% | / | 95.1 | +| grok-4-fast | 30 | 29.2% ± 7.2% | 53.3% | 16.7% | / | 74.7 | | claude-sonnet-4 | 30 | 27.5% ± 2.8% | 50.0% | 6.7% | $29 | 193.1 | | o4-mini | 30 | 25.0% ± 2.9% | 36.7% | 13.3% | $11.78 | 263.6 | +| deepseek-v3-1-terminus-thinking | 30 | 24.2% ± 10.1% | 43.3% | 6.7% | / | 773.4 | | gemini-2-5-pro | 30 | 24.2% ± 3.6% | 43.3% | 10.0% | $19.61 | 126.1 | | claude-sonnet-4-low | 30 | 23.3% ± 4.7% | 36.7% | 13.3% | $95.98 | 176.0 | | grok-code-fast-1 | 30 | 23.3% ± 7.4% | 40.0% | 10.0% | $1.76 | 75.5 | @@ -63,7 +68,9 @@ Generated: 2025-09-18T12:47:26.868861 | gpt-4-1 | 30 | 12.5% ± 1.4% | 20.0% | 3.3% | $9.07 | 41.0 | | gpt-5-nano-low | 30 | 12.5% ± 3.6% | 30.0% | 3.3% | $1.19 | 129.1 | | gpt-5-mini-low | 30 | 12.5% ± 4.9% | 33.3% | 3.3% | $1.13 | 67.7 | +| deepseek-v3-1-terminus | 30 | 10.8% ± 4.9% | 20.0% | 3.3% | / | 179.8 | | qwen-3-max | 30 | 10.8% ± 1.4% | 13.3% | 10.0% | $14.54 | 133.9 | +| glm-4-6 | 30 | 10.0% ± 3.3% | 16.7% | 3.3% | / | 175.8 | | gemini-2-5-flash | 30 | 8.3% ± 1.7% | 13.3% | 6.7% | $1.18 | 62.1 | | glm-4-5 | 30 | 7.5% ± 1.4% | 13.3% | 3.3% | $2.08 | 130.2 | | gpt-5-nano-medium | 30 | 6.7% ± 5.3% | 16.7% | 0.0% | $0.93 | 129.5 | @@ -78,13 +85,14 @@ Generated: 2025-09-18T12:47:26.868861 |-------|-------------|--------------------|----------|----------|---------------------|--------------------| | gpt-5-high | 23 | 50.0% ± 2.2% | 60.9% | 34.8% | $34.73 | 1083.5 | | gpt-5-medium | 23 | 47.8% ± 8.1% | 65.2% | 17.4% | $23.7 | 456.6 | +| claude-sonnet-4-5 | 23 | 29.3% ± 8.9% | 43.5% | 17.4% | / | 220.0 | | claude-sonnet-4-high | 23 | 28.3% ± 2.2% | 43.5% | 21.7% | $58.39 | 170.0 | | gpt-5-low | 23 | 27.2% ± 1.9% | 39.1% | 17.4% | $22.31 | 268.3 | | claude-sonnet-4-low | 23 | 25.0% ± 3.6% | 34.8% | 21.7% | $57.93 | 173.6 | | glm-4-5 | 23 | 22.8% ± 6.4% | 34.8% | 13.0% | $3.77 | 153.5 | -| claude-opus-4-1 | 23 | 21.7% ± 0.0% | / | / | $224.18 | 390.2 | | qwen-3-coder-plus | 23 | 19.6% ± 6.5% | 34.8% | 13.0% | $9.2 | 320.4 | | gpt-5-mini-high | 23 | 19.6% ± 2.2% | 34.8% | 8.7% | $6.92 | 338.1 | +| glm-4-6 | 23 | 19.6% ± 2.2% | 30.4% | 13.0% | / | 214.0 | | gpt-5-mini-medium | 23 | 18.5% ± 7.8% | 34.8% | 4.3% | $3.89 | 127.6 | | claude-sonnet-4 | 23 | 16.3% ± 5.7% | 30.4% | 8.7% | $49.61 | 196.5 | | kimi-k2-0905 | 23 | 16.3% ± 1.9% | 26.1% | 8.7% | $14.21 | 780.8 | @@ -93,6 +101,8 @@ Generated: 2025-09-18T12:47:26.868861 | qwen-3-max | 23 | 14.1% ± 3.6% | 17.4% | 4.3% | $37.56 | 181.5 | | grok-4 | 23 | 14.1% ± 3.6% | 21.7% | 8.7% | $56.18 | 269.0 | | o4-mini | 23 | 14.1% ± 6.4% | 26.1% | 4.3% | $13.79 | 248.8 | +| grok-4-fast | 23 | 13.0% ± 3.1% | 21.7% | 0.0% | / | 143.1 | +| deepseek-v3-1-terminus-thinking | 23 | 10.9% ± 4.9% | 21.7% | 0.0% | / | 702.3 | | kimi-k2-0711 | 23 | 10.9% ± 2.2% | 13.0% | 4.3% | $5.37 | 205.0 | | deepseek-chat | 23 | 9.8% ± 1.9% | 13.0% | 8.7% | $4.75 | 194.0 | | gemini-2-5-pro | 23 | 9.8% ± 1.9% | 21.7% | 0.0% | $11.96 | 91.3 | @@ -102,6 +112,7 @@ Generated: 2025-09-18T12:47:26.868861 | gpt-4-1 | 23 | 7.6% ± 1.9% | 8.7% | 4.3% | $20.97 | 90.2 | | gpt-5-nano-medium | 23 | 7.6% ± 1.9% | 13.0% | 0.0% | $1.11 | 187.3 | | gpt-4-1-mini | 23 | 6.5% ± 6.5% | 17.4% | 0.0% | $4.35 | 83.1 | +| deepseek-v3-1-terminus | 23 | 5.4% ± 7.1% | 17.4% | 0.0% | / | 231.9 | | gpt-oss-120b | 23 | 4.3% ± 3.1% | 8.7% | 0.0% | $0.14 | 24.0 | | gpt-5-nano-low | 23 | 0.0% ± 0.0% | 0.0% | 0.0% | $0.29 | 57.7 | | gpt-4-1-nano | 23 | 0.0% ± 0.0% | 0.0% | 0.0% | $0.74 | 51.8 | @@ -113,9 +124,12 @@ Generated: 2025-09-18T12:47:26.868861 | gpt-5-high | 28 | 44.6% ± 1.8% | 60.7% | 21.4% | $28.98 | 1161.4 | | gpt-5-medium | 28 | 42.0% ± 3.0% | 50.0% | 32.1% | $21.98 | 661.8 | | gpt-5-low | 28 | 36.6% ± 7.7% | 53.6% | 14.3% | $23.26 | 559.6 | -| claude-opus-4-1 | 28 | 35.7% ± 0.0% | / | / | $276.24 | 294.2 | +| claude-sonnet-4-5 | 28 | 25.0% ± 7.6% | 46.4% | 3.6% | / | 165.0 | +| glm-4-6 | 28 | 25.0% ± 4.4% | 39.3% | 14.3% | / | 299.3 | | o3 | 28 | 24.1% ± 3.9% | 46.4% | 7.1% | $14.72 | 171.4 | +| deepseek-v3-1-terminus-thinking | 28 | 22.3% ± 3.0% | 39.3% | 3.6% | / | 919.4 | | claude-sonnet-4-low | 28 | 22.3% ± 3.0% | 32.1% | 7.1% | $105.36 | 220.0 | +| deepseek-v3-1-terminus | 28 | 22.3% ± 8.1% | 39.3% | 7.1% | / | 252.5 | | glm-4-5 | 28 | 21.4% ± 2.5% | 32.1% | 10.7% | $5.97 | 222.2 | | claude-sonnet-4 | 28 | 21.4% ± 5.1% | 39.3% | 7.1% | $56.1 | 193.2 | | gpt-5-mini-high | 28 | 20.5% ± 13.0% | 42.9% | 7.1% | $10.62 | 522.6 | @@ -133,6 +147,7 @@ Generated: 2025-09-18T12:47:26.868861 | gemini-2-5-pro | 28 | 4.5% ± 3.0% | 7.1% | 0.0% | $17.9 | 102.4 | | gpt-5-nano-medium | 28 | 3.6% ± 0.0% | 3.6% | 3.6% | $0.65 | 170.5 | | gpt-oss-120b | 28 | 3.6% ± 2.5% | 14.3% | 0.0% | $0.15 | 34.0 | +| grok-4-fast | 28 | 3.6% ± 0.0% | 3.6% | 3.6% | / | 152.7 | | grok-code-fast-1 | 28 | 2.7% ± 1.6% | 3.6% | 0.0% | $3.45 | 334.1 | | grok-4 | 28 | 2.7% ± 1.6% | 3.6% | 0.0% | $62.48 | 554.0 | | gpt-4-1-mini | 28 | 1.8% ± 1.8% | 3.6% | 0.0% | $3 | 59.1 | @@ -150,18 +165,22 @@ Generated: 2025-09-18T12:47:26.868861 | grok-4 | 25 | 35.0% ± 7.7% | 48.0% | 20.0% | $97.36 | 277.2 | | qwen-3-coder-plus | 25 | 30.0% ± 4.5% | 48.0% | 8.0% | $14.31 | 680.0 | | kimi-k2-0905 | 25 | 30.0% ± 6.0% | 40.0% | 20.0% | $20.51 | 380.6 | +| claude-sonnet-4-5 | 25 | 27.0% ± 5.9% | 36.0% | 16.0% | / | 175.4 | +| grok-4-fast | 25 | 27.0% ± 3.3% | 40.0% | 16.0% | / | 105.8 | | claude-sonnet-4 | 25 | 26.0% ± 6.0% | 36.0% | 8.0% | $94.47 | 278.7 | | claude-sonnet-4-high | 25 | 26.0% ± 2.0% | 28.0% | 24.0% | $154.28 | 261.9 | | grok-code-fast-1 | 25 | 25.0% ± 1.7% | 36.0% | 8.0% | $6.06 | 119.5 | -| claude-opus-4-1 | 25 | 24.0% ± 0.0% | / | / | $435.18 | 395.2 | | claude-sonnet-4-low | 25 | 22.0% ± 3.5% | 28.0% | 20.0% | $157.11 | 239.1 | +| glm-4-6 | 25 | 19.0% ± 5.2% | 32.0% | 12.0% | / | 270.3 | | o3 | 25 | 15.0% ± 5.2% | 32.0% | 8.0% | $28.71 | 153.9 | | gemini-2-5-pro | 25 | 15.0% ± 1.7% | 32.0% | 4.0% | $108.12 | 177.7 | | gpt-5-mini-high | 25 | 15.0% ± 5.2% | 32.0% | 4.0% | $15.42 | 365.6 | | glm-4-5 | 25 | 13.0% ± 3.3% | 20.0% | 4.0% | $4.9 | 165.6 | +| deepseek-v3-1-terminus | 25 | 13.0% ± 1.7% | 20.0% | 8.0% | / | 329.9 | | kimi-k2-0711 | 25 | 13.0% ± 3.3% | 16.0% | 8.0% | $11.17 | 221.4 | | gpt-5-mini-medium | 25 | 12.0% ± 6.3% | 24.0% | 4.0% | $11.77 | 216.0 | | o4-mini | 25 | 12.0% ± 2.8% | 28.0% | 0.0% | $25.71 | 530.6 | +| deepseek-v3-1-terminus-thinking | 25 | 9.0% ± 1.7% | 20.0% | 0.0% | / | 775.8 | | gpt-4-1 | 25 | 8.0% ± 2.8% | 12.0% | 4.0% | $43.16 | 92.2 | | qwen-3-max | 25 | 8.0% ± 0.0% | 12.0% | 4.0% | $69.1 | 417.7 | | deepseek-chat | 25 | 7.0% ± 3.3% | 16.0% | 0.0% | $11.78 | 288.3 | @@ -185,6 +204,9 @@ Generated: 2025-09-18T12:47:26.868861 | gpt-5-mini-medium | 21 | 61.9% ± 5.8% | 90.5% | 28.6% | $1 | 96.5 | | grok-4 | 21 | 58.3% ± 7.8% | 81.0% | 38.1% | $14.32 | 204.3 | | claude-sonnet-4 | 21 | 53.6% ± 6.2% | 71.4% | 38.1% | $23.24 | 239.5 | +| grok-4-fast | 21 | 52.4% ± 8.9% | 81.0% | 28.6% | / | 71.8 | +| glm-4-6 | 21 | 51.2% ± 2.1% | 71.4% | 28.6% | / | 263.1 | +| claude-sonnet-4-5 | 21 | 50.0% ± 4.1% | 66.7% | 38.1% | / | 241.8 | | claude-sonnet-4-high | 21 | 50.0% ± 7.1% | 66.7% | 38.1% | $29.68 | 165.1 | | claude-sonnet-4-low | 21 | 48.8% ± 7.0% | 71.4% | 33.3% | $44.57 | 186.2 | | qwen-3-coder-plus | 21 | 47.6% ± 5.8% | 61.9% | 38.1% | $2.5 | 140.9 | @@ -192,9 +214,10 @@ Generated: 2025-09-18T12:47:26.868861 | kimi-k2-0905 | 21 | 47.6% ± 4.8% | 66.7% | 28.6% | $5.84 | 517.3 | | qwen-3-max | 21 | 44.0% ± 2.1% | 52.4% | 38.1% | $5.46 | 159.6 | | deepseek-chat | 21 | 42.9% ± 7.5% | 61.9% | 28.6% | $3.89 | 355.6 | +| deepseek-v3-1-terminus-thinking | 21 | 41.7% ± 7.8% | 61.9% | 19.1% | / | 418.5 | | kimi-k2-0711 | 21 | 40.5% ± 7.9% | 71.4% | 28.6% | $3.55 | 248.4 | | o3 | 21 | 36.9% ± 4.0% | 66.7% | 14.3% | $3.46 | 75.6 | -| claude-opus-4-1 | 21 | 33.3% ± 0.0% | / | / | $97.54 | 515.4 | +| deepseek-v3-1-terminus | 21 | 33.3% ± 19.9% | 57.1% | 0.0% | / | 240.7 | | gemini-2-5-pro | 21 | 26.2% ± 7.9% | 47.6% | 9.5% | $4.89 | 93.5 | | gpt-5-nano-medium | 21 | 15.5% ± 5.2% | 28.6% | 4.8% | $0.3 | 129.1 | | glm-4-5 | 21 | 14.3% ± 7.5% | 23.8% | 0.0% | $1.56 | 158.1 | diff --git a/model_results/claude-opus-4-1/build_your_own_x__find_commit_date.json b/model_results/claude-opus-4-1/build_your_own_x__find_commit_date.json deleted file mode 100644 index 4f9f23b0..00000000 --- a/model_results/claude-opus-4-1/build_your_own_x__find_commit_date.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "build_your_own_x__find_commit_date", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 215448 tokens > 200000 maximum (tid: 2025082317403183736962859888900)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 101.42097783088684, - "token_usage": { - "input_tokens": 258338, - "output_tokens": 680, - "total_tokens": 259018, - "reasoning_tokens": null - }, - "turn_count": 5 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/build_your_own_x__find_rag_commit.json b/model_results/claude-opus-4-1/build_your_own_x__find_rag_commit.json deleted file mode 100644 index 627460ac..00000000 --- a/model_results/claude-opus-4-1/build_your_own_x__find_rag_commit.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "build_your_own_x__find_rag_commit", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 215298 tokens > 200000 maximum (tid: 202508231742415653545798495849)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 71.28984642028809, - "token_usage": { - "input_tokens": 100338, - "output_tokens": 599, - "total_tokens": 100937, - "reasoning_tokens": null - }, - "turn_count": 5 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/chinook__customer_data_migration.json b/model_results/claude-opus-4-1/chinook__customer_data_migration.json deleted file mode 100644 index f653d44d..00000000 --- a/model_results/claude-opus-4-1/chinook__customer_data_migration.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "chinook__customer_data_migration", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 1022.2541089057922, - "token_usage": { - "input_tokens": 386018, - "output_tokens": 18977, - "total_tokens": 404995, - "reasoning_tokens": null - }, - "turn_count": 19 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/chinook__employee_hierarchy_management.json b/model_results/claude-opus-4-1/chinook__employee_hierarchy_management.json deleted file mode 100644 index a8fe7234..00000000 --- a/model_results/claude-opus-4-1/chinook__employee_hierarchy_management.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "chinook__employee_hierarchy_management", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 322.3665874004364, - "token_usage": { - "input_tokens": 110287, - "output_tokens": 5976, - "total_tokens": 116263, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/chinook__sales_and_music_charts.json b/model_results/claude-opus-4-1/chinook__sales_and_music_charts.json deleted file mode 100644 index f71d36ae..00000000 --- a/model_results/claude-opus-4-1/chinook__sales_and_music_charts.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "chinook__sales_and_music_charts", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 424.17409896850586, - "token_usage": { - "input_tokens": 199197, - "output_tokens": 7297, - "total_tokens": 206494, - "reasoning_tokens": null - }, - "turn_count": 29 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/claude-code__automated_changelog_generation.json b/model_results/claude-opus-4-1/claude-code__automated_changelog_generation.json deleted file mode 100644 index 168dc213..00000000 --- a/model_results/claude-opus-4-1/claude-code__automated_changelog_generation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "claude-code__automated_changelog_generation", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 486.5178973674774, - "token_usage": { - "input_tokens": 551409, - "output_tokens": 6538, - "total_tokens": 557947, - "reasoning_tokens": null - }, - "turn_count": 10 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/claude-code__claude_collaboration_analysis.json b/model_results/claude-opus-4-1/claude-code__claude_collaboration_analysis.json deleted file mode 100644 index 84527343..00000000 --- a/model_results/claude-opus-4-1/claude-code__claude_collaboration_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "claude-code__claude_collaboration_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 207434 tokens > 200000 maximum (tid: 2025082320150166052510531035171)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 59.63302779197693, - "token_usage": { - "input_tokens": 214092, - "output_tokens": 271, - "total_tokens": 214363, - "reasoning_tokens": null - }, - "turn_count": 2 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/claude-code__critical_issue_hotfix_workflow.json b/model_results/claude-opus-4-1/claude-code__critical_issue_hotfix_workflow.json deleted file mode 100644 index cb28f690..00000000 --- a/model_results/claude-opus-4-1/claude-code__critical_issue_hotfix_workflow.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "claude-code__critical_issue_hotfix_workflow", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 325.1222264766693, - "token_usage": { - "input_tokens": 355340, - "output_tokens": 4063, - "total_tokens": 359403, - "reasoning_tokens": null - }, - "turn_count": 11 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/claude-code__feature_commit_tracking.json b/model_results/claude-opus-4-1/claude-code__feature_commit_tracking.json deleted file mode 100644 index aec45fda..00000000 --- a/model_results/claude-opus-4-1/claude-code__feature_commit_tracking.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "claude-code__feature_commit_tracking", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 205177 tokens > 200000 maximum (tid: 2025082318031772744949032239935)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 78.38062977790833, - "token_usage": { - "input_tokens": 243270, - "output_tokens": 492, - "total_tokens": 243762, - "reasoning_tokens": null - }, - "turn_count": 4 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/claude-code__label_color_standardization.json b/model_results/claude-opus-4-1/claude-code__label_color_standardization.json deleted file mode 100644 index a9d6d1a8..00000000 --- a/model_results/claude-opus-4-1/claude-code__label_color_standardization.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "claude-code__label_color_standardization", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 337.7997958660126, - "token_usage": { - "input_tokens": 651976, - "output_tokens": 3885, - "total_tokens": 655861, - "reasoning_tokens": null - }, - "turn_count": 13 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/company_in_a_box__employee_onboarding.json b/model_results/claude-opus-4-1/company_in_a_box__employee_onboarding.json deleted file mode 100644 index f1e6cb66..00000000 --- a/model_results/claude-opus-4-1/company_in_a_box__employee_onboarding.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "company_in_a_box__employee_onboarding", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 482.5655288696289, - "token_usage": { - "input_tokens": 400292, - "output_tokens": 7548, - "total_tokens": 407840, - "reasoning_tokens": null - }, - "turn_count": 15 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/company_in_a_box__goals_restructure.json b/model_results/claude-opus-4-1/company_in_a_box__goals_restructure.json deleted file mode 100644 index 2ed59454..00000000 --- a/model_results/claude-opus-4-1/company_in_a_box__goals_restructure.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "company_in_a_box__goals_restructure", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 286.6263744831085, - "token_usage": { - "input_tokens": 374244, - "output_tokens": 3049, - "total_tokens": 377293, - "reasoning_tokens": null - }, - "turn_count": 19 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/company_in_a_box__quarterly_review_dashboard.json b/model_results/claude-opus-4-1/company_in_a_box__quarterly_review_dashboard.json deleted file mode 100644 index 36e33d9c..00000000 --- a/model_results/claude-opus-4-1/company_in_a_box__quarterly_review_dashboard.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "company_in_a_box__quarterly_review_dashboard", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 497.55545115470886, - "token_usage": { - "input_tokens": 824570, - "output_tokens": 6513, - "total_tokens": 831083, - "reasoning_tokens": null - }, - "turn_count": 25 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/computer_science_student_dashboard__code_snippets_go.json b/model_results/claude-opus-4-1/computer_science_student_dashboard__code_snippets_go.json deleted file mode 100644 index 084bb18e..00000000 --- a/model_results/claude-opus-4-1/computer_science_student_dashboard__code_snippets_go.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "computer_science_student_dashboard__code_snippets_go", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 244.7666482925415, - "token_usage": { - "input_tokens": 169301, - "output_tokens": 3924, - "total_tokens": 173225, - "reasoning_tokens": null - }, - "turn_count": 11 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/computer_science_student_dashboard__courses_internships_relation.json b/model_results/claude-opus-4-1/computer_science_student_dashboard__courses_internships_relation.json deleted file mode 100644 index 4c865a46..00000000 --- a/model_results/claude-opus-4-1/computer_science_student_dashboard__courses_internships_relation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "computer_science_student_dashboard__courses_internships_relation", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 363.53436517715454, - "token_usage": { - "input_tokens": 420966, - "output_tokens": 4170, - "total_tokens": 425136, - "reasoning_tokens": null - }, - "turn_count": 21 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/computer_science_student_dashboard__study_session_tracker.json b/model_results/claude-opus-4-1/computer_science_student_dashboard__study_session_tracker.json deleted file mode 100644 index 569486a4..00000000 --- a/model_results/claude-opus-4-1/computer_science_student_dashboard__study_session_tracker.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "computer_science_student_dashboard__study_session_tracker", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 275.5879247188568, - "token_usage": { - "input_tokens": 330908, - "output_tokens": 3221, - "total_tokens": 334129, - "reasoning_tokens": null - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/desktop__music_report.json b/model_results/claude-opus-4-1/desktop__music_report.json deleted file mode 100644 index b862e216..00000000 --- a/model_results/claude-opus-4-1/desktop__music_report.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "desktop__music_report", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 98.89563918113708, - "token_usage": { - "input_tokens": 25612, - "output_tokens": 1527, - "total_tokens": 27139, - "reasoning_tokens": null - }, - "turn_count": 8 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/desktop__project_management.json b/model_results/claude-opus-4-1/desktop__project_management.json deleted file mode 100644 index a6d7f04a..00000000 --- a/model_results/claude-opus-4-1/desktop__project_management.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "desktop__project_management", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 559.8005208969116, - "token_usage": { - "input_tokens": 552323, - "output_tokens": 9255, - "total_tokens": 561578, - "reasoning_tokens": null - }, - "turn_count": 51 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/desktop__timeline_extraction.json b/model_results/claude-opus-4-1/desktop__timeline_extraction.json deleted file mode 100644 index 10bdee55..00000000 --- a/model_results/claude-opus-4-1/desktop__timeline_extraction.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "desktop__timeline_extraction", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 240.9384047985077, - "token_usage": { - "input_tokens": 188507, - "output_tokens": 3860, - "total_tokens": 192367, - "reasoning_tokens": null - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/desktop_template__budget_computation.json b/model_results/claude-opus-4-1/desktop_template__budget_computation.json deleted file mode 100644 index b0eb9bef..00000000 --- a/model_results/claude-opus-4-1/desktop_template__budget_computation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "desktop_template__budget_computation", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 125.44478368759155, - "token_usage": { - "input_tokens": 35389, - "output_tokens": 2139, - "total_tokens": 37528, - "reasoning_tokens": null - }, - "turn_count": 7 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/desktop_template__contact_information.json b/model_results/claude-opus-4-1/desktop_template__contact_information.json deleted file mode 100644 index 75e16598..00000000 --- a/model_results/claude-opus-4-1/desktop_template__contact_information.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "desktop_template__contact_information", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 161.22254180908203, - "token_usage": { - "input_tokens": 54985, - "output_tokens": 2735, - "total_tokens": 57720, - "reasoning_tokens": null - }, - "turn_count": 10 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/desktop_template__file_arrangement.json b/model_results/claude-opus-4-1/desktop_template__file_arrangement.json deleted file mode 100644 index e2104d71..00000000 --- a/model_results/claude-opus-4-1/desktop_template__file_arrangement.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "desktop_template__file_arrangement", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 581.3442401885986, - "token_usage": { - "input_tokens": 587381, - "output_tokens": 9091, - "total_tokens": 596472, - "reasoning_tokens": null - }, - "turn_count": 53 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/dvdrental__customer_analysis_fix.json b/model_results/claude-opus-4-1/dvdrental__customer_analysis_fix.json deleted file mode 100644 index 3606113f..00000000 --- a/model_results/claude-opus-4-1/dvdrental__customer_analysis_fix.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "dvdrental__customer_analysis_fix", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 225.17450332641602, - "token_usage": { - "input_tokens": 56978, - "output_tokens": 4207, - "total_tokens": 61185, - "reasoning_tokens": null - }, - "turn_count": 11 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/dvdrental__customer_analytics_optimization.json b/model_results/claude-opus-4-1/dvdrental__customer_analytics_optimization.json deleted file mode 100644 index 67322ba0..00000000 --- a/model_results/claude-opus-4-1/dvdrental__customer_analytics_optimization.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "dvdrental__customer_analytics_optimization", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 263.71051263809204, - "token_usage": { - "input_tokens": 362694, - "output_tokens": 4122, - "total_tokens": 366816, - "reasoning_tokens": null - }, - "turn_count": 14 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/dvdrental__film_inventory_management.json b/model_results/claude-opus-4-1/dvdrental__film_inventory_management.json deleted file mode 100644 index 950ee9b7..00000000 --- a/model_results/claude-opus-4-1/dvdrental__film_inventory_management.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "dvdrental__film_inventory_management", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 201.0764443874359, - "token_usage": { - "input_tokens": 57692, - "output_tokens": 3526, - "total_tokens": 61218, - "reasoning_tokens": null - }, - "turn_count": 13 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/easyr1__advanced_branch_strategy.json b/model_results/claude-opus-4-1/easyr1__advanced_branch_strategy.json deleted file mode 100644 index 18c8fa78..00000000 --- a/model_results/claude-opus-4-1/easyr1__advanced_branch_strategy.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "easyr1__advanced_branch_strategy", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 198.91717386245728, - "token_usage": { - "input_tokens": 235867, - "output_tokens": 2372, - "total_tokens": 238239, - "reasoning_tokens": null - }, - "turn_count": 10 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/easyr1__config_parameter_audit.json b/model_results/claude-opus-4-1/easyr1__config_parameter_audit.json deleted file mode 100644 index 64e1c02a..00000000 --- a/model_results/claude-opus-4-1/easyr1__config_parameter_audit.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "easyr1__config_parameter_audit", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 208744 tokens > 200000 maximum (tid: 2025082318264011634326121831836)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 577.8364017009735, - "token_usage": { - "input_tokens": 3295742, - "output_tokens": 2949, - "total_tokens": 3298691, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/easyr1__performance_regression_investigation.json b/model_results/claude-opus-4-1/easyr1__performance_regression_investigation.json deleted file mode 100644 index 4a70b5a6..00000000 --- a/model_results/claude-opus-4-1/easyr1__performance_regression_investigation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "easyr1__performance_regression_investigation", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 390.20820927619934, - "token_usage": { - "input_tokens": 450998, - "output_tokens": 5172, - "total_tokens": 456170, - "reasoning_tokens": null - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/easyr1__qwen3_issue_management.json b/model_results/claude-opus-4-1/easyr1__qwen3_issue_management.json deleted file mode 100644 index 07d55193..00000000 --- a/model_results/claude-opus-4-1/easyr1__qwen3_issue_management.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "easyr1__qwen3_issue_management", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 288.1276867389679, - "token_usage": { - "input_tokens": 653543, - "output_tokens": 2775, - "total_tokens": 656318, - "reasoning_tokens": null - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/employees__employee_demographics_report.json b/model_results/claude-opus-4-1/employees__employee_demographics_report.json deleted file mode 100644 index 5fc18888..00000000 --- a/model_results/claude-opus-4-1/employees__employee_demographics_report.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "employees__employee_demographics_report", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 518.9978232383728, - "token_usage": { - "input_tokens": 234688, - "output_tokens": 9902, - "total_tokens": 244590, - "reasoning_tokens": null - }, - "turn_count": 29 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/employees__employee_performance_analysis.json b/model_results/claude-opus-4-1/employees__employee_performance_analysis.json deleted file mode 100644 index c5e7c12d..00000000 --- a/model_results/claude-opus-4-1/employees__employee_performance_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "employees__employee_performance_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 658.864099740982, - "token_usage": { - "input_tokens": 313303, - "output_tokens": 12804, - "total_tokens": 326107, - "reasoning_tokens": null - }, - "turn_count": 34 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/employees__employee_project_tracking.json b/model_results/claude-opus-4-1/employees__employee_project_tracking.json deleted file mode 100644 index f0648754..00000000 --- a/model_results/claude-opus-4-1/employees__employee_project_tracking.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "employees__employee_project_tracking", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 450.58818912506104, - "token_usage": { - "input_tokens": 213055, - "output_tokens": 7961, - "total_tokens": 221016, - "reasoning_tokens": null - }, - "turn_count": 30 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/employees__employee_retention_analysis.json b/model_results/claude-opus-4-1/employees__employee_retention_analysis.json deleted file mode 100644 index c2d423d3..00000000 --- a/model_results/claude-opus-4-1/employees__employee_retention_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "employees__employee_retention_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 758.7352375984192, - "token_usage": { - "input_tokens": 375875, - "output_tokens": 15307, - "total_tokens": 391182, - "reasoning_tokens": null - }, - "turn_count": 31 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/employees__executive_dashboard_automation.json b/model_results/claude-opus-4-1/employees__executive_dashboard_automation.json deleted file mode 100644 index 2d17ef0b..00000000 --- a/model_results/claude-opus-4-1/employees__executive_dashboard_automation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "employees__executive_dashboard_automation", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 552.2760052680969, - "token_usage": { - "input_tokens": 304039, - "output_tokens": 10634, - "total_tokens": 314673, - "reasoning_tokens": null - }, - "turn_count": 31 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/employees__management_structure_analysis.json b/model_results/claude-opus-4-1/employees__management_structure_analysis.json deleted file mode 100644 index 4c91a6a8..00000000 --- a/model_results/claude-opus-4-1/employees__management_structure_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "employees__management_structure_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 534.903430223465, - "token_usage": { - "input_tokens": 273476, - "output_tokens": 10140, - "total_tokens": 283616, - "reasoning_tokens": null - }, - "turn_count": 28 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/eval_web__cloudflare_turnstile_challenge.json b/model_results/claude-opus-4-1/eval_web__cloudflare_turnstile_challenge.json deleted file mode 100644 index 1d4525e8..00000000 --- a/model_results/claude-opus-4-1/eval_web__cloudflare_turnstile_challenge.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "eval_web__cloudflare_turnstile_challenge", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 565.0432152748108, - "token_usage": { - "input_tokens": 548500, - "output_tokens": 7703, - "total_tokens": 556203, - "reasoning_tokens": null - }, - "turn_count": 25 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/eval_web__extraction_table.json b/model_results/claude-opus-4-1/eval_web__extraction_table.json deleted file mode 100644 index b589036b..00000000 --- a/model_results/claude-opus-4-1/eval_web__extraction_table.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "eval_web__extraction_table", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 151.304541349411, - "token_usage": { - "input_tokens": 33113, - "output_tokens": 2322, - "total_tokens": 35435, - "reasoning_tokens": null - }, - "turn_count": 4 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/file_context__duplicates_searching.json b/model_results/claude-opus-4-1/file_context__duplicates_searching.json deleted file mode 100644 index ebe03137..00000000 --- a/model_results/claude-opus-4-1/file_context__duplicates_searching.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "file_context__duplicates_searching", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 189.84558725357056, - "token_usage": { - "input_tokens": 87748, - "output_tokens": 3673, - "total_tokens": 91421, - "reasoning_tokens": null - }, - "turn_count": 14 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/file_context__file_merging.json b/model_results/claude-opus-4-1/file_context__file_merging.json deleted file mode 100644 index 5ee86d7a..00000000 --- a/model_results/claude-opus-4-1/file_context__file_merging.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "file_context__file_merging", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 91.51880502700806, - "token_usage": { - "input_tokens": 16470, - "output_tokens": 1823, - "total_tokens": 18293, - "reasoning_tokens": null - }, - "turn_count": 5 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/file_context__file_splitting.json b/model_results/claude-opus-4-1/file_context__file_splitting.json deleted file mode 100644 index c618e9f5..00000000 --- a/model_results/claude-opus-4-1/file_context__file_splitting.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "file_context__file_splitting", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 183.54871487617493, - "token_usage": { - "input_tokens": 77149, - "output_tokens": 3075, - "total_tokens": 80224, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/file_context__pattern_matching.json b/model_results/claude-opus-4-1/file_context__pattern_matching.json deleted file mode 100644 index 755cdd58..00000000 --- a/model_results/claude-opus-4-1/file_context__pattern_matching.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "file_context__pattern_matching", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 130.5777690410614, - "token_usage": { - "input_tokens": 27455, - "output_tokens": 2421, - "total_tokens": 29876, - "reasoning_tokens": null - }, - "turn_count": 7 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/file_context__uppercase.json b/model_results/claude-opus-4-1/file_context__uppercase.json deleted file mode 100644 index 8702e7ec..00000000 --- a/model_results/claude-opus-4-1/file_context__uppercase.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "file_context__uppercase", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 195.87972903251648, - "token_usage": { - "input_tokens": 85121, - "output_tokens": 3663, - "total_tokens": 88784, - "reasoning_tokens": null - }, - "turn_count": 17 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/file_property__size_classification.json b/model_results/claude-opus-4-1/file_property__size_classification.json deleted file mode 100644 index f1545f35..00000000 --- a/model_results/claude-opus-4-1/file_property__size_classification.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "file_property__size_classification", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 158.52210807800293, - "token_usage": { - "input_tokens": 66506, - "output_tokens": 2839, - "total_tokens": 69345, - "reasoning_tokens": null - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/file_property__time_classification.json b/model_results/claude-opus-4-1/file_property__time_classification.json deleted file mode 100644 index 357d6644..00000000 --- a/model_results/claude-opus-4-1/file_property__time_classification.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "file_property__time_classification", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 301.9879250526428, - "token_usage": { - "input_tokens": 187540, - "output_tokens": 5139, - "total_tokens": 192679, - "reasoning_tokens": null - }, - "turn_count": 32 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/folder_structure__structure_analysis.json b/model_results/claude-opus-4-1/folder_structure__structure_analysis.json deleted file mode 100644 index 36d989b7..00000000 --- a/model_results/claude-opus-4-1/folder_structure__structure_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "folder_structure__structure_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 146.45751810073853, - "token_usage": { - "input_tokens": 167036, - "output_tokens": 1881, - "total_tokens": 168917, - "reasoning_tokens": null - }, - "turn_count": 13 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/folder_structure__structure_mirror.json b/model_results/claude-opus-4-1/folder_structure__structure_mirror.json deleted file mode 100644 index b4e155b2..00000000 --- a/model_results/claude-opus-4-1/folder_structure__structure_mirror.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "folder_structure__structure_mirror", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 363.83149099349976, - "token_usage": { - "input_tokens": 354411, - "output_tokens": 5571, - "total_tokens": 359982, - "reasoning_tokens": null - }, - "turn_count": 33 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/harmony__fix_conflict.json b/model_results/claude-opus-4-1/harmony__fix_conflict.json deleted file mode 100644 index 9e9cc425..00000000 --- a/model_results/claude-opus-4-1/harmony__fix_conflict.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "harmony__fix_conflict", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 382.32449102401733, - "token_usage": { - "input_tokens": 869307, - "output_tokens": 4839, - "total_tokens": 874146, - "reasoning_tokens": null - }, - "turn_count": 14 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/harmony__issue_pr_commit_workflow.json b/model_results/claude-opus-4-1/harmony__issue_pr_commit_workflow.json deleted file mode 100644 index 7ee5cac9..00000000 --- a/model_results/claude-opus-4-1/harmony__issue_pr_commit_workflow.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "harmony__issue_pr_commit_workflow", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 267.2391390800476, - "token_usage": { - "input_tokens": 274885, - "output_tokens": 3532, - "total_tokens": 278417, - "reasoning_tokens": null - }, - "turn_count": 11 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/harmony__issue_tagging_pr_closure.json b/model_results/claude-opus-4-1/harmony__issue_tagging_pr_closure.json deleted file mode 100644 index 8a9a1fca..00000000 --- a/model_results/claude-opus-4-1/harmony__issue_tagging_pr_closure.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "harmony__issue_tagging_pr_closure", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 395.47656083106995, - "token_usage": { - "input_tokens": 453282, - "output_tokens": 4998, - "total_tokens": 458280, - "reasoning_tokens": null - }, - "turn_count": 15 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/harmony__multi_branch_commit_aggregation.json b/model_results/claude-opus-4-1/harmony__multi_branch_commit_aggregation.json deleted file mode 100644 index 4d2f84e4..00000000 --- a/model_results/claude-opus-4-1/harmony__multi_branch_commit_aggregation.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "harmony__multi_branch_commit_aggregation", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 597.5980651378632, - "token_usage": { - "input_tokens": 1328632, - "output_tokens": 7930, - "total_tokens": 1336562, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/harmony__release_management_workflow.json b/model_results/claude-opus-4-1/harmony__release_management_workflow.json deleted file mode 100644 index 27c39c15..00000000 --- a/model_results/claude-opus-4-1/harmony__release_management_workflow.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "harmony__release_management_workflow", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 565.036545753479, - "token_usage": { - "input_tokens": 1036903, - "output_tokens": 8321, - "total_tokens": 1045224, - "reasoning_tokens": 0 - }, - "turn_count": 17 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/it_trouble_shooting_hub__asset_retirement_migration.json b/model_results/claude-opus-4-1/it_trouble_shooting_hub__asset_retirement_migration.json deleted file mode 100644 index d3a691dd..00000000 --- a/model_results/claude-opus-4-1/it_trouble_shooting_hub__asset_retirement_migration.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "it_trouble_shooting_hub__asset_retirement_migration", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 336.1930365562439, - "token_usage": { - "input_tokens": 511670, - "output_tokens": 3064, - "total_tokens": 514734, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/it_trouble_shooting_hub__security_audit_ticket.json b/model_results/claude-opus-4-1/it_trouble_shooting_hub__security_audit_ticket.json deleted file mode 100644 index 4611182e..00000000 --- a/model_results/claude-opus-4-1/it_trouble_shooting_hub__security_audit_ticket.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "it_trouble_shooting_hub__security_audit_ticket", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 185.3166160583496, - "token_usage": { - "input_tokens": 215873, - "output_tokens": 2016, - "total_tokens": 217889, - "reasoning_tokens": null - }, - "turn_count": 8 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/it_trouble_shooting_hub__verification_expired_update.json b/model_results/claude-opus-4-1/it_trouble_shooting_hub__verification_expired_update.json deleted file mode 100644 index ffe43c31..00000000 --- a/model_results/claude-opus-4-1/it_trouble_shooting_hub__verification_expired_update.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "it_trouble_shooting_hub__verification_expired_update", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 284.5214612483978, - "token_usage": { - "input_tokens": 991980, - "output_tokens": 3993, - "total_tokens": 995973, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/japan_travel_planner__daily_itinerary_overview.json b/model_results/claude-opus-4-1/japan_travel_planner__daily_itinerary_overview.json deleted file mode 100644 index 14e982fa..00000000 --- a/model_results/claude-opus-4-1/japan_travel_planner__daily_itinerary_overview.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "japan_travel_planner__daily_itinerary_overview", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 127.18816041946411, - "token_usage": { - "input_tokens": 369809, - "output_tokens": 3078, - "total_tokens": 372887, - "reasoning_tokens": null - }, - "turn_count": 5 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/japan_travel_planner__packing_progress_summary.json b/model_results/claude-opus-4-1/japan_travel_planner__packing_progress_summary.json deleted file mode 100644 index cc47a76e..00000000 --- a/model_results/claude-opus-4-1/japan_travel_planner__packing_progress_summary.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "japan_travel_planner__packing_progress_summary", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 277.22349977493286, - "token_usage": { - "input_tokens": 1953987, - "output_tokens": 4197, - "total_tokens": 1958184, - "reasoning_tokens": null - }, - "turn_count": 21 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/japan_travel_planner__remove_osaka_itinerary.json b/model_results/claude-opus-4-1/japan_travel_planner__remove_osaka_itinerary.json deleted file mode 100644 index 633e669d..00000000 --- a/model_results/claude-opus-4-1/japan_travel_planner__remove_osaka_itinerary.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "japan_travel_planner__remove_osaka_itinerary", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 94.12291431427002, - "token_usage": { - "input_tokens": 792892, - "output_tokens": 1289, - "total_tokens": 794181, - "reasoning_tokens": null - }, - "turn_count": 8 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/japan_travel_planner__restaurant_expenses_sync.json b/model_results/claude-opus-4-1/japan_travel_planner__restaurant_expenses_sync.json deleted file mode 100644 index 8e4afcd7..00000000 --- a/model_results/claude-opus-4-1/japan_travel_planner__restaurant_expenses_sync.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "japan_travel_planner__restaurant_expenses_sync", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 125.08707022666931, - "token_usage": { - "input_tokens": 255917, - "output_tokens": 2431, - "total_tokens": 258348, - "reasoning_tokens": null - }, - "turn_count": 10 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/legal_document__dispute_review.json b/model_results/claude-opus-4-1/legal_document__dispute_review.json deleted file mode 100644 index a4f5c3da..00000000 --- a/model_results/claude-opus-4-1/legal_document__dispute_review.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "legal_document__dispute_review", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 124.05476307868958, - "token_usage": { - "input_tokens": 144729, - "output_tokens": 1415, - "total_tokens": 146144, - "reasoning_tokens": null - }, - "turn_count": 7 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/legal_document__individual_comments.json b/model_results/claude-opus-4-1/legal_document__individual_comments.json deleted file mode 100644 index 77cb9809..00000000 --- a/model_results/claude-opus-4-1/legal_document__individual_comments.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "legal_document__individual_comments", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 156.1454041004181, - "token_usage": { - "input_tokens": 193146, - "output_tokens": 1875, - "total_tokens": 195021, - "reasoning_tokens": null - }, - "turn_count": 7 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/legal_document__solution_tracing.json b/model_results/claude-opus-4-1/legal_document__solution_tracing.json deleted file mode 100644 index 24d6f268..00000000 --- a/model_results/claude-opus-4-1/legal_document__solution_tracing.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "legal_document__solution_tracing", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 143.57952189445496, - "token_usage": { - "input_tokens": 294946, - "output_tokens": 1614, - "total_tokens": 296560, - "reasoning_tokens": null - }, - "turn_count": 8 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/lego__consistency_enforcement.json b/model_results/claude-opus-4-1/lego__consistency_enforcement.json deleted file mode 100644 index e3e1ad80..00000000 --- a/model_results/claude-opus-4-1/lego__consistency_enforcement.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "lego__consistency_enforcement", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 393.2954270839691, - "token_usage": { - "input_tokens": 172724, - "output_tokens": 7067, - "total_tokens": 179791, - "reasoning_tokens": null - }, - "turn_count": 25 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/lego__database_security_policies.json b/model_results/claude-opus-4-1/lego__database_security_policies.json deleted file mode 100644 index d0ec9ec5..00000000 --- a/model_results/claude-opus-4-1/lego__database_security_policies.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "lego__database_security_policies", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 888.4085941314697, - "token_usage": { - "input_tokens": 353413, - "output_tokens": 18779, - "total_tokens": 372192, - "reasoning_tokens": null - }, - "turn_count": 30 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/lego__transactional_inventory_transfer.json b/model_results/claude-opus-4-1/lego__transactional_inventory_transfer.json deleted file mode 100644 index 6f0ef81f..00000000 --- a/model_results/claude-opus-4-1/lego__transactional_inventory_transfer.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "lego__transactional_inventory_transfer", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 601.5841164588928, - "token_usage": { - "input_tokens": 304864, - "output_tokens": 11731, - "total_tokens": 316595, - "reasoning_tokens": null - }, - "turn_count": 29 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/mcpmark-cicd__deployment_status_workflow.json b/model_results/claude-opus-4-1/mcpmark-cicd__deployment_status_workflow.json deleted file mode 100644 index 3bace85c..00000000 --- a/model_results/claude-opus-4-1/mcpmark-cicd__deployment_status_workflow.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "mcpmark-cicd__deployment_status_workflow", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 2233.8593094348907, - "token_usage": { - "input_tokens": 403936, - "output_tokens": 51557, - "total_tokens": 455493, - "reasoning_tokens": 0 - }, - "turn_count": 20 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/mcpmark-cicd__issue_management_workflow.json b/model_results/claude-opus-4-1/mcpmark-cicd__issue_management_workflow.json deleted file mode 100644 index b8e8f5ab..00000000 --- a/model_results/claude-opus-4-1/mcpmark-cicd__issue_management_workflow.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "mcpmark-cicd__issue_management_workflow", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 531.0920813083649, - "token_usage": { - "input_tokens": 551218, - "output_tokens": 8782, - "total_tokens": 560000, - "reasoning_tokens": 0 - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/mcpmark-cicd__linting_ci_workflow.json b/model_results/claude-opus-4-1/mcpmark-cicd__linting_ci_workflow.json deleted file mode 100644 index cb1b686c..00000000 --- a/model_results/claude-opus-4-1/mcpmark-cicd__linting_ci_workflow.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "mcpmark-cicd__linting_ci_workflow", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 174.83519554138184, - "token_usage": { - "input_tokens": 155317, - "output_tokens": 2781, - "total_tokens": 158098, - "reasoning_tokens": null - }, - "turn_count": 7 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/mcpmark-cicd__pr_automation_workflow.json b/model_results/claude-opus-4-1/mcpmark-cicd__pr_automation_workflow.json deleted file mode 100644 index ccab9078..00000000 --- a/model_results/claude-opus-4-1/mcpmark-cicd__pr_automation_workflow.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "mcpmark-cicd__pr_automation_workflow", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 420.4544110298157, - "token_usage": { - "input_tokens": 117291, - "output_tokens": 9198, - "total_tokens": 126489, - "reasoning_tokens": 0 - }, - "turn_count": 6 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/missing-semester__assign_contributor_labels.json b/model_results/claude-opus-4-1/missing-semester__assign_contributor_labels.json deleted file mode 100644 index fa609163..00000000 --- a/model_results/claude-opus-4-1/missing-semester__assign_contributor_labels.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "missing-semester__assign_contributor_labels", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 203212 tokens > 200000 maximum (tid: 202508231933251014129401802239)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 110.90351128578186, - "token_usage": { - "input_tokens": 370537, - "output_tokens": 829, - "total_tokens": 371366, - "reasoning_tokens": null - }, - "turn_count": 3 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/missing-semester__find_legacy_name.json b/model_results/claude-opus-4-1/missing-semester__find_legacy_name.json deleted file mode 100644 index dae8f07b..00000000 --- a/model_results/claude-opus-4-1/missing-semester__find_legacy_name.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "missing-semester__find_legacy_name", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 203780 tokens > 200000 maximum (tid: 2025082319353588124051470957573)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 58.568087577819824, - "token_usage": { - "input_tokens": 192528, - "output_tokens": 290, - "total_tokens": 192818, - "reasoning_tokens": null - }, - "turn_count": 2 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/missing-semester__find_salient_file.json b/model_results/claude-opus-4-1/missing-semester__find_salient_file.json deleted file mode 100644 index f63eb4f7..00000000 --- a/model_results/claude-opus-4-1/missing-semester__find_salient_file.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "github", - "task": "missing-semester__find_salient_file", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 202667 tokens > 200000 maximum (tid: 2025082319420652335404707673054)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 321.5846378803253, - "token_usage": { - "input_tokens": 1509742, - "output_tokens": 1380, - "total_tokens": 1511122, - "reasoning_tokens": null - }, - "turn_count": 9 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/online_resume__layout_adjustment.json b/model_results/claude-opus-4-1/online_resume__layout_adjustment.json deleted file mode 100644 index c44c2d00..00000000 --- a/model_results/claude-opus-4-1/online_resume__layout_adjustment.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "online_resume__layout_adjustment", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 425.41325330734253, - "token_usage": { - "input_tokens": 3192034, - "output_tokens": 6401, - "total_tokens": 3198435, - "reasoning_tokens": null - }, - "turn_count": 40 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/online_resume__projects_section_update.json b/model_results/claude-opus-4-1/online_resume__projects_section_update.json deleted file mode 100644 index 1a5eb8fa..00000000 --- a/model_results/claude-opus-4-1/online_resume__projects_section_update.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "online_resume__projects_section_update", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 181.69771671295166, - "token_usage": { - "input_tokens": 847570, - "output_tokens": 2903, - "total_tokens": 850473, - "reasoning_tokens": null - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/online_resume__skills_development_tracker.json b/model_results/claude-opus-4-1/online_resume__skills_development_tracker.json deleted file mode 100644 index 705a0700..00000000 --- a/model_results/claude-opus-4-1/online_resume__skills_development_tracker.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "online_resume__skills_development_tracker", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 247.4410319328308, - "token_usage": { - "input_tokens": 622238, - "output_tokens": 4324, - "total_tokens": 626562, - "reasoning_tokens": null - }, - "turn_count": 19 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/online_resume__work_history_addition.json b/model_results/claude-opus-4-1/online_resume__work_history_addition.json deleted file mode 100644 index e5fcb370..00000000 --- a/model_results/claude-opus-4-1/online_resume__work_history_addition.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "online_resume__work_history_addition", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 211.4523344039917, - "token_usage": { - "input_tokens": 438871, - "output_tokens": 3223, - "total_tokens": 442094, - "reasoning_tokens": null - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/papers__author_folders.json b/model_results/claude-opus-4-1/papers__author_folders.json deleted file mode 100644 index 5e7a3d8f..00000000 --- a/model_results/claude-opus-4-1/papers__author_folders.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "papers__author_folders", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 656.0237100124359, - "token_usage": { - "input_tokens": 1366936, - "output_tokens": 11172, - "total_tokens": 1378108, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/papers__find_math_paper.json b/model_results/claude-opus-4-1/papers__find_math_paper.json deleted file mode 100644 index 9bf4d8a2..00000000 --- a/model_results/claude-opus-4-1/papers__find_math_paper.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "papers__find_math_paper", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 140.44481205940247, - "token_usage": { - "input_tokens": 421671, - "output_tokens": 1378, - "total_tokens": 423049, - "reasoning_tokens": null - }, - "turn_count": 6 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/papers__organize_legacy_papers.json b/model_results/claude-opus-4-1/papers__organize_legacy_papers.json deleted file mode 100644 index c188e9ac..00000000 --- a/model_results/claude-opus-4-1/papers__organize_legacy_papers.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "papers__organize_legacy_papers", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 212839 tokens > 200000 maximum (tid: 202508231627049510938847528203)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 140.9607548713684, - "token_usage": { - "input_tokens": 40663, - "output_tokens": 1636, - "total_tokens": 42299, - "reasoning_tokens": null - }, - "turn_count": 9 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/python_roadmap__expert_level_lessons.json b/model_results/claude-opus-4-1/python_roadmap__expert_level_lessons.json deleted file mode 100644 index 434194e0..00000000 --- a/model_results/claude-opus-4-1/python_roadmap__expert_level_lessons.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "python_roadmap__expert_level_lessons", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 386.3332588672638, - "token_usage": { - "input_tokens": 605665, - "output_tokens": 4641, - "total_tokens": 610306, - "reasoning_tokens": null - }, - "turn_count": 20 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/python_roadmap__learning_metrics_dashboard.json b/model_results/claude-opus-4-1/python_roadmap__learning_metrics_dashboard.json deleted file mode 100644 index 4b5bbebe..00000000 --- a/model_results/claude-opus-4-1/python_roadmap__learning_metrics_dashboard.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "python_roadmap__learning_metrics_dashboard", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 272.8607907295227, - "token_usage": { - "input_tokens": 364570, - "output_tokens": 4207, - "total_tokens": 368777, - "reasoning_tokens": null - }, - "turn_count": 11 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/reddit__ai_data_analyst.json b/model_results/claude-opus-4-1/reddit__ai_data_analyst.json deleted file mode 100644 index 6b8c6703..00000000 --- a/model_results/claude-opus-4-1/reddit__ai_data_analyst.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "reddit__ai_data_analyst", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 205071 tokens > 200000 maximum (tid: 2025082316070915174961593657145)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 393.66253876686096, - "token_usage": { - "input_tokens": 1307145, - "output_tokens": 2321, - "total_tokens": 1309466, - "reasoning_tokens": null - }, - "turn_count": 20 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/reddit__budget_europe_travel.json b/model_results/claude-opus-4-1/reddit__budget_europe_travel.json deleted file mode 100644 index a70b60ce..00000000 --- a/model_results/claude-opus-4-1/reddit__budget_europe_travel.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "reddit__budget_europe_travel", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 534.0608282089233, - "token_usage": { - "input_tokens": 953450, - "output_tokens": 5090, - "total_tokens": 958540, - "reasoning_tokens": null - }, - "turn_count": 32 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/reddit__buyitforlife_research.json b/model_results/claude-opus-4-1/reddit__buyitforlife_research.json deleted file mode 100644 index 74073f01..00000000 --- a/model_results/claude-opus-4-1/reddit__buyitforlife_research.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "reddit__buyitforlife_research", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 417.5229127407074, - "token_usage": { - "input_tokens": 1596799, - "output_tokens": 2703, - "total_tokens": 1599502, - "reasoning_tokens": null - }, - "turn_count": 17 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/reddit__llm_research_summary.json b/model_results/claude-opus-4-1/reddit__llm_research_summary.json deleted file mode 100644 index bc1e4c15..00000000 --- a/model_results/claude-opus-4-1/reddit__llm_research_summary.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "reddit__llm_research_summary", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 403.86750411987305, - "token_usage": { - "input_tokens": 984735, - "output_tokens": 3636, - "total_tokens": 988371, - "reasoning_tokens": null - }, - "turn_count": 20 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/reddit__movie_reviewer_analysis.json b/model_results/claude-opus-4-1/reddit__movie_reviewer_analysis.json deleted file mode 100644 index 61ea8452..00000000 --- a/model_results/claude-opus-4-1/reddit__movie_reviewer_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "reddit__movie_reviewer_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 357.2769422531128, - "token_usage": { - "input_tokens": 930347, - "output_tokens": 3140, - "total_tokens": 933487, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/reddit__nba_statistics_analysis.json b/model_results/claude-opus-4-1/reddit__nba_statistics_analysis.json deleted file mode 100644 index 4df53892..00000000 --- a/model_results/claude-opus-4-1/reddit__nba_statistics_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "reddit__nba_statistics_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 203281 tokens > 200000 maximum (tid: 2025082316455186802473497037315)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 377.93063974380493, - "token_usage": { - "input_tokens": 1224076, - "output_tokens": 2161, - "total_tokens": 1226237, - "reasoning_tokens": null - }, - "turn_count": 19 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/reddit__routine_tracker_forum.json b/model_results/claude-opus-4-1/reddit__routine_tracker_forum.json deleted file mode 100644 index aea851a3..00000000 --- a/model_results/claude-opus-4-1/reddit__routine_tracker_forum.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "reddit__routine_tracker_forum", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 205530 tokens > 200000 maximum (tid: 2025082316522923998404997677254)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 346.12610816955566, - "token_usage": { - "input_tokens": 1220576, - "output_tokens": 2063, - "total_tokens": 1222639, - "reasoning_tokens": null - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/security__rls_business_access.json b/model_results/claude-opus-4-1/security__rls_business_access.json deleted file mode 100644 index c2d56e7b..00000000 --- a/model_results/claude-opus-4-1/security__rls_business_access.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "security__rls_business_access", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 1202.568341255188, - "token_usage": { - "input_tokens": 699229, - "output_tokens": 22522, - "total_tokens": 721751, - "reasoning_tokens": null - }, - "turn_count": 41 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/security__user_permission_audit.json b/model_results/claude-opus-4-1/security__user_permission_audit.json deleted file mode 100644 index bcffd63c..00000000 --- a/model_results/claude-opus-4-1/security__user_permission_audit.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "security__user_permission_audit", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 403.78105878829956, - "token_usage": { - "input_tokens": 174179, - "output_tokens": 6917, - "total_tokens": 181096, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/self_assessment__faq_column_layout.json b/model_results/claude-opus-4-1/self_assessment__faq_column_layout.json deleted file mode 100644 index e5a503e5..00000000 --- a/model_results/claude-opus-4-1/self_assessment__faq_column_layout.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "self_assessment__faq_column_layout", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 214.57503175735474, - "token_usage": { - "input_tokens": 294092, - "output_tokens": 2935, - "total_tokens": 297027, - "reasoning_tokens": null - }, - "turn_count": 11 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/self_assessment__hyperfocus_analysis_report.json b/model_results/claude-opus-4-1/self_assessment__hyperfocus_analysis_report.json deleted file mode 100644 index 699ae8b9..00000000 --- a/model_results/claude-opus-4-1/self_assessment__hyperfocus_analysis_report.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "self_assessment__hyperfocus_analysis_report", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 251.0410759449005, - "token_usage": { - "input_tokens": 264610, - "output_tokens": 4542, - "total_tokens": 269152, - "reasoning_tokens": null - }, - "turn_count": 8 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/self_assessment__numbered_list_emojis.json b/model_results/claude-opus-4-1/self_assessment__numbered_list_emojis.json deleted file mode 100644 index a4bdbc5f..00000000 --- a/model_results/claude-opus-4-1/self_assessment__numbered_list_emojis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "self_assessment__numbered_list_emojis", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 256.4135408401489, - "token_usage": { - "input_tokens": 452760, - "output_tokens": 2723, - "total_tokens": 455483, - "reasoning_tokens": null - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping__advanced_product_analysis.json b/model_results/claude-opus-4-1/shopping__advanced_product_analysis.json deleted file mode 100644 index 1d22b915..00000000 --- a/model_results/claude-opus-4-1/shopping__advanced_product_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping__advanced_product_analysis", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 452.6573419570923, - "token_usage": { - "input_tokens": 1438842, - "output_tokens": 3138, - "total_tokens": 1441980, - "reasoning_tokens": null - }, - "turn_count": 20 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping__gaming_accessories_analysis.json b/model_results/claude-opus-4-1/shopping__gaming_accessories_analysis.json deleted file mode 100644 index 31512fe9..00000000 --- a/model_results/claude-opus-4-1/shopping__gaming_accessories_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping__gaming_accessories_analysis", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 869.9663565158844, - "token_usage": { - "input_tokens": 3654616, - "output_tokens": 4436, - "total_tokens": 3659052, - "reasoning_tokens": null - }, - "turn_count": 37 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping__health_routine_optimization.json b/model_results/claude-opus-4-1/shopping__health_routine_optimization.json deleted file mode 100644 index 02e1ead8..00000000 --- a/model_results/claude-opus-4-1/shopping__health_routine_optimization.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping__health_routine_optimization", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 516.1689138412476, - "token_usage": { - "input_tokens": 1719919, - "output_tokens": 3654, - "total_tokens": 1723573, - "reasoning_tokens": null - }, - "turn_count": 23 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping__holiday_baking_competition.json b/model_results/claude-opus-4-1/shopping__holiday_baking_competition.json deleted file mode 100644 index 28b2c56e..00000000 --- a/model_results/claude-opus-4-1/shopping__holiday_baking_competition.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping__holiday_baking_competition", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 202755 tokens > 200000 maximum (tid: 2025082317382978892586071506376)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 647.6959841251373, - "token_usage": { - "input_tokens": 2520129, - "output_tokens": 3457, - "total_tokens": 2523586, - "reasoning_tokens": null - }, - "turn_count": 26 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping__multi_category_budget_analysis.json b/model_results/claude-opus-4-1/shopping__multi_category_budget_analysis.json deleted file mode 100644 index 06fdec9c..00000000 --- a/model_results/claude-opus-4-1/shopping__multi_category_budget_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping__multi_category_budget_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 202860 tokens > 200000 maximum (tid: 202508231750551844765427451839)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 690.4746689796448, - "token_usage": { - "input_tokens": 2680142, - "output_tokens": 3645, - "total_tokens": 2683787, - "reasoning_tokens": null - }, - "turn_count": 27 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping__printer_keyboard_search.json b/model_results/claude-opus-4-1/shopping__printer_keyboard_search.json deleted file mode 100644 index 3dcd33b0..00000000 --- a/model_results/claude-opus-4-1/shopping__printer_keyboard_search.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping__printer_keyboard_search", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 216.40940713882446, - "token_usage": { - "input_tokens": 474583, - "output_tokens": 1824, - "total_tokens": 476407, - "reasoning_tokens": null - }, - "turn_count": 10 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping__running_shoes_purchase.json b/model_results/claude-opus-4-1/shopping__running_shoes_purchase.json deleted file mode 100644 index 88cc0ea0..00000000 --- a/model_results/claude-opus-4-1/shopping__running_shoes_purchase.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping__running_shoes_purchase", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 258.12507224082947, - "token_usage": { - "input_tokens": 707798, - "output_tokens": 2069, - "total_tokens": 709867, - "reasoning_tokens": null - }, - "turn_count": 13 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping_admin__customer_segmentation_setup.json b/model_results/claude-opus-4-1/shopping_admin__customer_segmentation_setup.json deleted file mode 100644 index e3360fea..00000000 --- a/model_results/claude-opus-4-1/shopping_admin__customer_segmentation_setup.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping_admin__customer_segmentation_setup", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 301.27563405036926, - "token_usage": { - "input_tokens": 641339, - "output_tokens": 2500, - "total_tokens": 643839, - "reasoning_tokens": null - }, - "turn_count": 20 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping_admin__fitness_promotion_strategy.json b/model_results/claude-opus-4-1/shopping_admin__fitness_promotion_strategy.json deleted file mode 100644 index a2a7a805..00000000 --- a/model_results/claude-opus-4-1/shopping_admin__fitness_promotion_strategy.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping_admin__fitness_promotion_strategy", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 210361 tokens > 200000 maximum (tid: 202508231812093159235821112369)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 235.47408413887024, - "token_usage": { - "input_tokens": 675712, - "output_tokens": 1525, - "total_tokens": 677237, - "reasoning_tokens": null - }, - "turn_count": 12 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping_admin__marketing_customer_analysis.json b/model_results/claude-opus-4-1/shopping_admin__marketing_customer_analysis.json deleted file mode 100644 index ba03e78a..00000000 --- a/model_results/claude-opus-4-1/shopping_admin__marketing_customer_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping_admin__marketing_customer_analysis", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 522.9044888019562, - "token_usage": { - "input_tokens": 1620249, - "output_tokens": 3647, - "total_tokens": 1623896, - "reasoning_tokens": null - }, - "turn_count": 28 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping_admin__ny_expansion_analysis.json b/model_results/claude-opus-4-1/shopping_admin__ny_expansion_analysis.json deleted file mode 100644 index 64e3849a..00000000 --- a/model_results/claude-opus-4-1/shopping_admin__ny_expansion_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping_admin__ny_expansion_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 312.54434847831726, - "token_usage": { - "input_tokens": 588846, - "output_tokens": 2206, - "total_tokens": 591052, - "reasoning_tokens": null - }, - "turn_count": 17 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping_admin__products_sales_analysis.json b/model_results/claude-opus-4-1/shopping_admin__products_sales_analysis.json deleted file mode 100644 index 7c1ab1a8..00000000 --- a/model_results/claude-opus-4-1/shopping_admin__products_sales_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping_admin__products_sales_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 201145 tokens > 200000 maximum (tid: 2025082318301421594594847533538)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 115.03723692893982, - "token_usage": { - "input_tokens": 71720, - "output_tokens": 1140, - "total_tokens": 72860, - "reasoning_tokens": null - }, - "turn_count": 8 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping_admin__sales_inventory_analysis.json b/model_results/claude-opus-4-1/shopping_admin__sales_inventory_analysis.json deleted file mode 100644 index ddc4330c..00000000 --- a/model_results/claude-opus-4-1/shopping_admin__sales_inventory_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping_admin__sales_inventory_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 202753 tokens > 200000 maximum (tid: 2025082318332830080650111553282)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 166.58150815963745, - "token_usage": { - "input_tokens": 426336, - "output_tokens": 1046, - "total_tokens": 427382, - "reasoning_tokens": null - }, - "turn_count": 9 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/shopping_admin__search_filtering_operations.json b/model_results/claude-opus-4-1/shopping_admin__search_filtering_operations.json deleted file mode 100644 index 5f3652c8..00000000 --- a/model_results/claude-opus-4-1/shopping_admin__search_filtering_operations.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "shopping_admin__search_filtering_operations", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 633.6722922325134, - "token_usage": { - "input_tokens": 1615670, - "output_tokens": 4381, - "total_tokens": 1620051, - "reasoning_tokens": null - }, - "turn_count": 40 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/sports__baseball_player_analysis.json b/model_results/claude-opus-4-1/sports__baseball_player_analysis.json deleted file mode 100644 index efce9bd6..00000000 --- a/model_results/claude-opus-4-1/sports__baseball_player_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "sports__baseball_player_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 444.3078725337982, - "token_usage": { - "input_tokens": 158872, - "output_tokens": 9194, - "total_tokens": 168066, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/sports__participant_report_optimization.json b/model_results/claude-opus-4-1/sports__participant_report_optimization.json deleted file mode 100644 index db9be129..00000000 --- a/model_results/claude-opus-4-1/sports__participant_report_optimization.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "sports__participant_report_optimization", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 277.8676087856293, - "token_usage": { - "input_tokens": 123864, - "output_tokens": 4546, - "total_tokens": 128410, - "reasoning_tokens": null - }, - "turn_count": 22 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/sports__team_roster_management.json b/model_results/claude-opus-4-1/sports__team_roster_management.json deleted file mode 100644 index e4c5a437..00000000 --- a/model_results/claude-opus-4-1/sports__team_roster_management.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "sports__team_roster_management", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 325.1091368198395, - "token_usage": { - "input_tokens": 158775, - "output_tokens": 5538, - "total_tokens": 164313, - "reasoning_tokens": null - }, - "turn_count": 25 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/standard_operating_procedure__deployment_process_sop.json b/model_results/claude-opus-4-1/standard_operating_procedure__deployment_process_sop.json deleted file mode 100644 index bdcd5b4f..00000000 --- a/model_results/claude-opus-4-1/standard_operating_procedure__deployment_process_sop.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "standard_operating_procedure__deployment_process_sop", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 584.4173250198364, - "token_usage": { - "input_tokens": 1108203, - "output_tokens": 5896, - "total_tokens": 1114099, - "reasoning_tokens": null - }, - "turn_count": 36 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/standard_operating_procedure__section_organization.json b/model_results/claude-opus-4-1/standard_operating_procedure__section_organization.json deleted file mode 100644 index 4844ad84..00000000 --- a/model_results/claude-opus-4-1/standard_operating_procedure__section_organization.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "standard_operating_procedure__section_organization", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 361.242742061615, - "token_usage": { - "input_tokens": 501656, - "output_tokens": 4149, - "total_tokens": 505805, - "reasoning_tokens": null - }, - "turn_count": 24 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/student_database__duplicate_name.json b/model_results/claude-opus-4-1/student_database__duplicate_name.json deleted file mode 100644 index 4626af59..00000000 --- a/model_results/claude-opus-4-1/student_database__duplicate_name.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "student_database__duplicate_name", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 73.8231930732727, - "token_usage": { - "input_tokens": 13960, - "output_tokens": 1167, - "total_tokens": 15127, - "reasoning_tokens": null - }, - "turn_count": 4 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/student_database__english_talent.json b/model_results/claude-opus-4-1/student_database__english_talent.json deleted file mode 100644 index 5516495a..00000000 --- a/model_results/claude-opus-4-1/student_database__english_talent.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "student_database__english_talent", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 1082.4093272686005, - "token_usage": { - "input_tokens": 1498886, - "output_tokens": 21607, - "total_tokens": 1520493, - "reasoning_tokens": null - }, - "turn_count": 29 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/student_database__gradebased_score.json b/model_results/claude-opus-4-1/student_database__gradebased_score.json deleted file mode 100644 index 29c24ccc..00000000 --- a/model_results/claude-opus-4-1/student_database__gradebased_score.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "student_database__gradebased_score", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 691.0160908699036, - "token_usage": { - "input_tokens": 248633, - "output_tokens": 14246, - "total_tokens": 262879, - "reasoning_tokens": null - }, - "turn_count": 11 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/team_projects__priority_tasks_table.json b/model_results/claude-opus-4-1/team_projects__priority_tasks_table.json deleted file mode 100644 index 7166cf4e..00000000 --- a/model_results/claude-opus-4-1/team_projects__priority_tasks_table.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "team_projects__priority_tasks_table", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 200.5851604938507, - "token_usage": { - "input_tokens": 181232, - "output_tokens": 2920, - "total_tokens": 184152, - "reasoning_tokens": null - }, - "turn_count": 11 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/team_projects__swap_tasks.json b/model_results/claude-opus-4-1/team_projects__swap_tasks.json deleted file mode 100644 index 8eefc892..00000000 --- a/model_results/claude-opus-4-1/team_projects__swap_tasks.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "team_projects__swap_tasks", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 313.7170798778534, - "token_usage": { - "input_tokens": 524794, - "output_tokens": 3329, - "total_tokens": 528123, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/threestudio__code_locating.json b/model_results/claude-opus-4-1/threestudio__code_locating.json deleted file mode 100644 index 722abe5f..00000000 --- a/model_results/claude-opus-4-1/threestudio__code_locating.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "threestudio__code_locating", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 114.27874207496643, - "token_usage": { - "input_tokens": 47564, - "output_tokens": 1408, - "total_tokens": 48972, - "reasoning_tokens": null - }, - "turn_count": 10 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/threestudio__output_analysis.json b/model_results/claude-opus-4-1/threestudio__output_analysis.json deleted file mode 100644 index c7998fbe..00000000 --- a/model_results/claude-opus-4-1/threestudio__output_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "threestudio__output_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 152.56501817703247, - "token_usage": { - "input_tokens": 130219, - "output_tokens": 1628, - "total_tokens": 131847, - "reasoning_tokens": null - }, - "turn_count": 11 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/threestudio__requirements_completion.json b/model_results/claude-opus-4-1/threestudio__requirements_completion.json deleted file mode 100644 index 6f2a5111..00000000 --- a/model_results/claude-opus-4-1/threestudio__requirements_completion.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "threestudio__requirements_completion", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 254.11752915382385, - "token_usage": { - "input_tokens": 219559, - "output_tokens": 3564, - "total_tokens": 223123, - "reasoning_tokens": null - }, - "turn_count": 24 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/toronto_guide__change_color.json b/model_results/claude-opus-4-1/toronto_guide__change_color.json deleted file mode 100644 index 6ce2b074..00000000 --- a/model_results/claude-opus-4-1/toronto_guide__change_color.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "toronto_guide__change_color", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 542.5061004161835, - "token_usage": { - "input_tokens": 703574, - "output_tokens": 6298, - "total_tokens": 709872, - "reasoning_tokens": null - }, - "turn_count": 26 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/toronto_guide__weekend_adventure_planner.json b/model_results/claude-opus-4-1/toronto_guide__weekend_adventure_planner.json deleted file mode 100644 index 0fe1c465..00000000 --- a/model_results/claude-opus-4-1/toronto_guide__weekend_adventure_planner.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "notion", - "task": "toronto_guide__weekend_adventure_planner", - "runs": { - "run-1": { - "success": true, - "error_message": null, - "execution_time": 208.82200622558594, - "token_usage": { - "input_tokens": 151321, - "output_tokens": 3160, - "total_tokens": 154481, - "reasoning_tokens": null - }, - "turn_count": 10 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/vectors__dba_vector_analysis.json b/model_results/claude-opus-4-1/vectors__dba_vector_analysis.json deleted file mode 100644 index 49d8409e..00000000 --- a/model_results/claude-opus-4-1/vectors__dba_vector_analysis.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "postgres", - "task": "vectors__dba_vector_analysis", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 352.9074103832245, - "token_usage": { - "input_tokens": 441117, - "output_tokens": 8585, - "total_tokens": 449702, - "reasoning_tokens": null - }, - "turn_count": 27 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/votenet__dataset_comparison.json b/model_results/claude-opus-4-1/votenet__dataset_comparison.json deleted file mode 100644 index 83c45ae3..00000000 --- a/model_results/claude-opus-4-1/votenet__dataset_comparison.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "votenet__dataset_comparison", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 335.7183780670166, - "token_usage": { - "input_tokens": 567839, - "output_tokens": 4403, - "total_tokens": 572242, - "reasoning_tokens": null - }, - "turn_count": 16 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/votenet__debugging.json b/model_results/claude-opus-4-1/votenet__debugging.json deleted file mode 100644 index a7efad71..00000000 --- a/model_results/claude-opus-4-1/votenet__debugging.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "votenet__debugging", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 194.19709610939026, - "token_usage": { - "input_tokens": 90470, - "output_tokens": 2199, - "total_tokens": 92669, - "reasoning_tokens": null - }, - "turn_count": 13 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/votenet__requirements_writing.json b/model_results/claude-opus-4-1/votenet__requirements_writing.json deleted file mode 100644 index e3dc8528..00000000 --- a/model_results/claude-opus-4-1/votenet__requirements_writing.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "filesystem", - "task": "votenet__requirements_writing", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 244.1449418067932, - "token_usage": { - "input_tokens": 372125, - "output_tokens": 2990, - "total_tokens": 375115, - "reasoning_tokens": null - }, - "turn_count": 18 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/web_search__birth_of_arvinxu.json b/model_results/claude-opus-4-1/web_search__birth_of_arvinxu.json deleted file mode 100644 index 837c2fd3..00000000 --- a/model_results/claude-opus-4-1/web_search__birth_of_arvinxu.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "web_search__birth_of_arvinxu", - "runs": { - "run-1": { - "success": false, - "error_message": null, - "execution_time": 202.9762270450592, - "token_usage": { - "input_tokens": 356654, - "output_tokens": 1465, - "total_tokens": 358119, - "reasoning_tokens": null - }, - "turn_count": 8 - } - } -} \ No newline at end of file diff --git a/model_results/claude-opus-4-1/web_search__r1_arxiv.json b/model_results/claude-opus-4-1/web_search__r1_arxiv.json deleted file mode 100644 index 4d2197ce..00000000 --- a/model_results/claude-opus-4-1/web_search__r1_arxiv.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "model": "claude-opus-4-1", - "service": "playwright", - "task": "web_search__r1_arxiv", - "runs": { - "run-1": { - "success": false, - "error_message": "Error code: 400 - {'error': {'message': 'prompt is too long: 214131 tokens > 200000 maximum (tid: 2025082316214256910978319706814)', 'type': 'invalid_request_error', 'param': '', 'code': None}}", - "execution_time": 192.04109406471252, - "token_usage": { - "input_tokens": 660052, - "output_tokens": 835, - "total_tokens": 660887, - "reasoning_tokens": null - }, - "turn_count": 7 - } - } -} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/build_your_own_x__find_commit_date.json b/model_results/claude-sonnet-4-5/build_your_own_x__find_commit_date.json new file mode 100644 index 00000000..079c535b --- /dev/null +++ b/model_results/claude-sonnet-4-5/build_your_own_x__find_commit_date.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "build_your_own_x__find_commit_date", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=claude-sonnet-4.5\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers", + "execution_time": 9.048184156417847, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 152.6909122467041, + "token_usage": { + "input_tokens": 2193967, + "output_tokens": 2995, + "total_tokens": 2196962, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 147.16514229774475, + "token_usage": { + "input_tokens": 2050152, + "output_tokens": 2952, + "total_tokens": 2053104, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 73.4245445728302, + "token_usage": { + "input_tokens": 586966, + "output_tokens": 1706, + "total_tokens": 588672, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/build_your_own_x__find_rag_commit.json b/model_results/claude-sonnet-4-5/build_your_own_x__find_rag_commit.json new file mode 100644 index 00000000..15421733 --- /dev/null +++ b/model_results/claude-sonnet-4-5/build_your_own_x__find_rag_commit.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "build_your_own_x__find_rag_commit", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=claude-sonnet-4.5\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers", + "execution_time": 9.023174524307251, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 149.35746383666992, + "token_usage": { + "input_tokens": 2252545, + "output_tokens": 2737, + "total_tokens": 2255282, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 170.13444805145264, + "token_usage": { + "input_tokens": 2508309, + "output_tokens": 3253, + "total_tokens": 2511562, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 194.2477638721466, + "token_usage": { + "input_tokens": 2630412, + "output_tokens": 3921, + "total_tokens": 2634333, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/chinook__customer_data_migration.json b/model_results/claude-sonnet-4-5/chinook__customer_data_migration.json new file mode 100644 index 00000000..618b90ab --- /dev/null +++ b/model_results/claude-sonnet-4-5/chinook__customer_data_migration.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "chinook__customer_data_migration", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=claude-sonnet-4.5\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers", + "execution_time": 22.106448888778687, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-2": { + "success": false, + "error_message": "Execution timed out after 3600 seconds", + "execution_time": 3600.1552386283875, + "token_usage": { + "input_tokens": 1258126, + "output_tokens": 184097, + "total_tokens": 1442223, + "reasoning_tokens": 0 + }, + "turn_count": 59 + }, + "run-3": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 1394.781268119812, + "token_usage": { + "input_tokens": 2416116, + "output_tokens": 56721, + "total_tokens": 2472837, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-4": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 1213.39200425148, + "token_usage": { + "input_tokens": 2354137, + "output_tokens": 44442, + "total_tokens": 2398579, + "reasoning_tokens": 0 + }, + "turn_count": 100 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/chinook__employee_hierarchy_management.json b/model_results/claude-sonnet-4-5/chinook__employee_hierarchy_management.json new file mode 100644 index 00000000..1c6f4abf --- /dev/null +++ b/model_results/claude-sonnet-4-5/chinook__employee_hierarchy_management.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "chinook__employee_hierarchy_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 118.62710762023926, + "token_usage": { + "input_tokens": 272418, + "output_tokens": 4058, + "total_tokens": 276476, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 103.54423666000366, + "token_usage": { + "input_tokens": 248315, + "output_tokens": 3785, + "total_tokens": 252100, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 93.01032948493958, + "token_usage": { + "input_tokens": 168384, + "output_tokens": 4425, + "total_tokens": 172809, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 137.89153909683228, + "token_usage": { + "input_tokens": 246161, + "output_tokens": 3990, + "total_tokens": 250151, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/chinook__sales_and_music_charts.json b/model_results/claude-sonnet-4-5/chinook__sales_and_music_charts.json new file mode 100644 index 00000000..1d32535d --- /dev/null +++ b/model_results/claude-sonnet-4-5/chinook__sales_and_music_charts.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "chinook__sales_and_music_charts", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 119.53029990196228, + "token_usage": { + "input_tokens": 308218, + "output_tokens": 5732, + "total_tokens": 313950, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 101.49867391586304, + "token_usage": { + "input_tokens": 276512, + "output_tokens": 4542, + "total_tokens": 281054, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 93.56715822219849, + "token_usage": { + "input_tokens": 268102, + "output_tokens": 4216, + "total_tokens": 272318, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 101.90559959411621, + "token_usage": { + "input_tokens": 272276, + "output_tokens": 4433, + "total_tokens": 276709, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/claude-code__automated_changelog_generation.json b/model_results/claude-sonnet-4-5/claude-code__automated_changelog_generation.json new file mode 100644 index 00000000..ad70af94 --- /dev/null +++ b/model_results/claude-sonnet-4-5/claude-code__automated_changelog_generation.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "claude-code__automated_changelog_generation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 335.13299441337585, + "token_usage": { + "input_tokens": 424907, + "output_tokens": 14771, + "total_tokens": 439678, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 295.59840059280396, + "token_usage": { + "input_tokens": 426113, + "output_tokens": 13573, + "total_tokens": 439686, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 225.89520597457886, + "token_usage": { + "input_tokens": 475948, + "output_tokens": 9401, + "total_tokens": 485349, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1719.6133711338043, + "token_usage": { + "input_tokens": 1846401, + "output_tokens": 78909, + "total_tokens": 1925310, + "reasoning_tokens": 0 + }, + "turn_count": 33 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/claude-code__claude_collaboration_analysis.json b/model_results/claude-sonnet-4-5/claude-code__claude_collaboration_analysis.json new file mode 100644 index 00000000..fb021d79 --- /dev/null +++ b/model_results/claude-sonnet-4-5/claude-code__claude_collaboration_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "claude-code__claude_collaboration_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 80.06109738349915, + "token_usage": { + "input_tokens": 218564, + "output_tokens": 1587, + "total_tokens": 220151, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 137.86703252792358, + "token_usage": { + "input_tokens": 648856, + "output_tokens": 4738, + "total_tokens": 653594, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 56.66862773895264, + "token_usage": { + "input_tokens": 297445, + "output_tokens": 1765, + "total_tokens": 299210, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 47.77523708343506, + "token_usage": { + "input_tokens": 225802, + "output_tokens": 1607, + "total_tokens": 227409, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/claude-code__critical_issue_hotfix_workflow.json b/model_results/claude-sonnet-4-5/claude-code__critical_issue_hotfix_workflow.json new file mode 100644 index 00000000..afb27a75 --- /dev/null +++ b/model_results/claude-sonnet-4-5/claude-code__critical_issue_hotfix_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "claude-code__critical_issue_hotfix_workflow", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 173.35112285614014, + "token_usage": { + "input_tokens": 341396, + "output_tokens": 4934, + "total_tokens": 346330, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 136.18039441108704, + "token_usage": { + "input_tokens": 393881, + "output_tokens": 4392, + "total_tokens": 398273, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 126.43626260757446, + "token_usage": { + "input_tokens": 301671, + "output_tokens": 4326, + "total_tokens": 305997, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 117.63972783088684, + "token_usage": { + "input_tokens": 250105, + "output_tokens": 4189, + "total_tokens": 254294, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/claude-code__feature_commit_tracking.json b/model_results/claude-sonnet-4-5/claude-code__feature_commit_tracking.json new file mode 100644 index 00000000..f7528da7 --- /dev/null +++ b/model_results/claude-sonnet-4-5/claude-code__feature_commit_tracking.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "claude-code__feature_commit_tracking", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 120.03905081748962, + "token_usage": { + "input_tokens": 745728, + "output_tokens": 2635, + "total_tokens": 748363, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 114.45466208457947, + "token_usage": { + "input_tokens": 1330661, + "output_tokens": 2650, + "total_tokens": 1333311, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 154.2811348438263, + "token_usage": { + "input_tokens": 1274078, + "output_tokens": 3184, + "total_tokens": 1277262, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 91.4626739025116, + "token_usage": { + "input_tokens": 893881, + "output_tokens": 1977, + "total_tokens": 895858, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/claude-code__label_color_standardization.json b/model_results/claude-sonnet-4-5/claude-code__label_color_standardization.json new file mode 100644 index 00000000..91bac8df --- /dev/null +++ b/model_results/claude-sonnet-4-5/claude-code__label_color_standardization.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "claude-code__label_color_standardization", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 185.12812423706055, + "token_usage": { + "input_tokens": 859308, + "output_tokens": 4642, + "total_tokens": 863950, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 186.4637885093689, + "token_usage": { + "input_tokens": 1680481, + "output_tokens": 5281, + "total_tokens": 1685762, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 170.27613186836243, + "token_usage": { + "input_tokens": 872897, + "output_tokens": 5731, + "total_tokens": 878628, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 167.31954193115234, + "token_usage": { + "input_tokens": 1349621, + "output_tokens": 4746, + "total_tokens": 1354367, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/company_in_a_box__employee_onboarding.json b/model_results/claude-sonnet-4-5/company_in_a_box__employee_onboarding.json new file mode 100644 index 00000000..1eb0ee21 --- /dev/null +++ b/model_results/claude-sonnet-4-5/company_in_a_box__employee_onboarding.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "company_in_a_box__employee_onboarding", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=claude-sonnet-4.5\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers", + "execution_time": 7.384501695632935, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 177.6273341178894, + "token_usage": { + "input_tokens": 548642, + "output_tokens": 8107, + "total_tokens": 556749, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 202.14041090011597, + "token_usage": { + "input_tokens": 1340614, + "output_tokens": 6531, + "total_tokens": 1347145, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 199.804523229599, + "token_usage": { + "input_tokens": 1019675, + "output_tokens": 6257, + "total_tokens": 1025932, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/company_in_a_box__goals_restructure.json b/model_results/claude-sonnet-4-5/company_in_a_box__goals_restructure.json new file mode 100644 index 00000000..97214585 --- /dev/null +++ b/model_results/claude-sonnet-4-5/company_in_a_box__goals_restructure.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "company_in_a_box__goals_restructure", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=claude-sonnet-4.5\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers", + "execution_time": 7.180706739425659, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 106.1369080543518, + "token_usage": { + "input_tokens": 142522, + "output_tokens": 4559, + "total_tokens": 147081, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 109.21507000923157, + "token_usage": { + "input_tokens": 267423, + "output_tokens": 4115, + "total_tokens": 271538, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 138.61377835273743, + "token_usage": { + "input_tokens": 939219, + "output_tokens": 4143, + "total_tokens": 943362, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/company_in_a_box__quarterly_review_dashboard.json b/model_results/claude-sonnet-4-5/company_in_a_box__quarterly_review_dashboard.json new file mode 100644 index 00000000..d7c70fc0 --- /dev/null +++ b/model_results/claude-sonnet-4-5/company_in_a_box__quarterly_review_dashboard.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "company_in_a_box__quarterly_review_dashboard", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=claude-sonnet-4.5\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers", + "execution_time": 7.286128044128418, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 303.9621379375458, + "token_usage": { + "input_tokens": 1646641, + "output_tokens": 10655, + "total_tokens": 1657296, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 503.8980305194855, + "token_usage": { + "input_tokens": 2113595, + "output_tokens": 11382, + "total_tokens": 2124977, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 403.12762999534607, + "token_usage": { + "input_tokens": 3463729, + "output_tokens": 10265, + "total_tokens": 3473994, + "reasoning_tokens": 0 + }, + "turn_count": 40 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/computer_science_student_dashboard__code_snippets_go.json b/model_results/claude-sonnet-4-5/computer_science_student_dashboard__code_snippets_go.json new file mode 100644 index 00000000..00154691 --- /dev/null +++ b/model_results/claude-sonnet-4-5/computer_science_student_dashboard__code_snippets_go.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "computer_science_student_dashboard__code_snippets_go", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 189.04399418830872, + "token_usage": { + "input_tokens": 166723, + "output_tokens": 2885, + "total_tokens": 169608, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 75.76531982421875, + "token_usage": { + "input_tokens": 142324, + "output_tokens": 2939, + "total_tokens": 145263, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 76.72847747802734, + "token_usage": { + "input_tokens": 140643, + "output_tokens": 2756, + "total_tokens": 143399, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 114.53728413581848, + "token_usage": { + "input_tokens": 1042557, + "output_tokens": 2293, + "total_tokens": 1044850, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/computer_science_student_dashboard__courses_internships_relation.json b/model_results/claude-sonnet-4-5/computer_science_student_dashboard__courses_internships_relation.json new file mode 100644 index 00000000..54b904a2 --- /dev/null +++ b/model_results/claude-sonnet-4-5/computer_science_student_dashboard__courses_internships_relation.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "computer_science_student_dashboard__courses_internships_relation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 132.7680013179779, + "token_usage": { + "input_tokens": 244992, + "output_tokens": 3650, + "total_tokens": 248642, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 179.38031792640686, + "token_usage": { + "input_tokens": 446478, + "output_tokens": 6153, + "total_tokens": 452631, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 402.04554772377014, + "token_usage": { + "input_tokens": 1136128, + "output_tokens": 7014, + "total_tokens": 1143142, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 401.2056860923767, + "token_usage": { + "input_tokens": 1845772, + "output_tokens": 11382, + "total_tokens": 1857154, + "reasoning_tokens": 0 + }, + "turn_count": 30 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/computer_science_student_dashboard__study_session_tracker.json b/model_results/claude-sonnet-4-5/computer_science_student_dashboard__study_session_tracker.json new file mode 100644 index 00000000..8baf6d58 --- /dev/null +++ b/model_results/claude-sonnet-4-5/computer_science_student_dashboard__study_session_tracker.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "computer_science_student_dashboard__study_session_tracker", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 157.88507723808289, + "token_usage": { + "input_tokens": 241738, + "output_tokens": 4129, + "total_tokens": 245867, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 64.28976011276245, + "token_usage": { + "input_tokens": 112607, + "output_tokens": 2244, + "total_tokens": 114851, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 230.87277221679688, + "token_usage": { + "input_tokens": 644963, + "output_tokens": 7011, + "total_tokens": 651974, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 45.71245288848877, + "token_usage": { + "input_tokens": 82118, + "output_tokens": 1779, + "total_tokens": 83897, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/desktop__music_report.json b/model_results/claude-sonnet-4-5/desktop__music_report.json new file mode 100644 index 00000000..a0297d42 --- /dev/null +++ b/model_results/claude-sonnet-4-5/desktop__music_report.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "desktop__music_report", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=claude-sonnet-4.5\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers", + "execution_time": 7.522918462753296, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 49.86474108695984, + "token_usage": { + "input_tokens": 37772, + "output_tokens": 3309, + "total_tokens": 41081, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 48.36807107925415, + "token_usage": { + "input_tokens": 37653, + "output_tokens": 3298, + "total_tokens": 40951, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 48.838834285736084, + "token_usage": { + "input_tokens": 37709, + "output_tokens": 3270, + "total_tokens": 40979, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/desktop__project_management.json b/model_results/claude-sonnet-4-5/desktop__project_management.json new file mode 100644 index 00000000..cacdee70 --- /dev/null +++ b/model_results/claude-sonnet-4-5/desktop__project_management.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "desktop__project_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 167.9004487991333, + "token_usage": { + "input_tokens": 168532, + "output_tokens": 7416, + "total_tokens": 175948, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 119.9151623249054, + "token_usage": { + "input_tokens": 168190, + "output_tokens": 7133, + "total_tokens": 175323, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 113.30031991004944, + "token_usage": { + "input_tokens": 158346, + "output_tokens": 7538, + "total_tokens": 165884, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 130.4593415260315, + "token_usage": { + "input_tokens": 202823, + "output_tokens": 8231, + "total_tokens": 211054, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/desktop__timeline_extraction.json b/model_results/claude-sonnet-4-5/desktop__timeline_extraction.json new file mode 100644 index 00000000..ab6d9ab0 --- /dev/null +++ b/model_results/claude-sonnet-4-5/desktop__timeline_extraction.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "desktop__timeline_extraction", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 81.54736590385437, + "token_usage": { + "input_tokens": 119386, + "output_tokens": 2893, + "total_tokens": 122279, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 63.116132497787476, + "token_usage": { + "input_tokens": 104173, + "output_tokens": 3299, + "total_tokens": 107472, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 71.25612831115723, + "token_usage": { + "input_tokens": 160317, + "output_tokens": 4073, + "total_tokens": 164390, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 71.5654833316803, + "token_usage": { + "input_tokens": 140307, + "output_tokens": 4304, + "total_tokens": 144611, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/desktop_template__budget_computation.json b/model_results/claude-sonnet-4-5/desktop_template__budget_computation.json new file mode 100644 index 00000000..251aaf96 --- /dev/null +++ b/model_results/claude-sonnet-4-5/desktop_template__budget_computation.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "desktop_template__budget_computation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 90.00802803039551, + "token_usage": { + "input_tokens": 47006, + "output_tokens": 4058, + "total_tokens": 51064, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 57.92476272583008, + "token_usage": { + "input_tokens": 59880, + "output_tokens": 2560, + "total_tokens": 62440, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 60.530882358551025, + "token_usage": { + "input_tokens": 68601, + "output_tokens": 2830, + "total_tokens": 71431, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 53.02128624916077, + "token_usage": { + "input_tokens": 59940, + "output_tokens": 2356, + "total_tokens": 62296, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/desktop_template__contact_information.json b/model_results/claude-sonnet-4-5/desktop_template__contact_information.json new file mode 100644 index 00000000..c043f0dd --- /dev/null +++ b/model_results/claude-sonnet-4-5/desktop_template__contact_information.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "desktop_template__contact_information", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 87.12614560127258, + "token_usage": { + "input_tokens": 61464, + "output_tokens": 2747, + "total_tokens": 64211, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 71.39080858230591, + "token_usage": { + "input_tokens": 63198, + "output_tokens": 3322, + "total_tokens": 66520, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 75.84327030181885, + "token_usage": { + "input_tokens": 105610, + "output_tokens": 3283, + "total_tokens": 108893, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 67.84210777282715, + "token_usage": { + "input_tokens": 91793, + "output_tokens": 3251, + "total_tokens": 95044, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/desktop_template__file_arrangement.json b/model_results/claude-sonnet-4-5/desktop_template__file_arrangement.json new file mode 100644 index 00000000..e7af0d73 --- /dev/null +++ b/model_results/claude-sonnet-4-5/desktop_template__file_arrangement.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "desktop_template__file_arrangement", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 159.62848925590515, + "token_usage": { + "input_tokens": 142678, + "output_tokens": 5677, + "total_tokens": 148355, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 105.37904787063599, + "token_usage": { + "input_tokens": 150584, + "output_tokens": 5759, + "total_tokens": 156343, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 73.0366370677948, + "token_usage": { + "input_tokens": 93779, + "output_tokens": 4887, + "total_tokens": 98666, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 83.96731734275818, + "token_usage": { + "input_tokens": 124307, + "output_tokens": 5346, + "total_tokens": 129653, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/dvdrental__customer_analysis_fix.json b/model_results/claude-sonnet-4-5/dvdrental__customer_analysis_fix.json new file mode 100644 index 00000000..6ca8588c --- /dev/null +++ b/model_results/claude-sonnet-4-5/dvdrental__customer_analysis_fix.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "dvdrental__customer_analysis_fix", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 135.74979424476624, + "token_usage": { + "input_tokens": 263437, + "output_tokens": 6774, + "total_tokens": 270211, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 84.19785523414612, + "token_usage": { + "input_tokens": 141888, + "output_tokens": 4135, + "total_tokens": 146023, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 122.44764375686646, + "token_usage": { + "input_tokens": 242075, + "output_tokens": 5825, + "total_tokens": 247900, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 286.97572350502014, + "token_usage": { + "input_tokens": 205801, + "output_tokens": 6004, + "total_tokens": 211805, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/dvdrental__customer_analytics_optimization.json b/model_results/claude-sonnet-4-5/dvdrental__customer_analytics_optimization.json new file mode 100644 index 00000000..604a091a --- /dev/null +++ b/model_results/claude-sonnet-4-5/dvdrental__customer_analytics_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "dvdrental__customer_analytics_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 112.24652767181396, + "token_usage": { + "input_tokens": 220008, + "output_tokens": 5309, + "total_tokens": 225317, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 111.87647032737732, + "token_usage": { + "input_tokens": 149256, + "output_tokens": 5916, + "total_tokens": 155172, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 167.59008932113647, + "token_usage": { + "input_tokens": 254440, + "output_tokens": 7424, + "total_tokens": 261864, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 91.85426902770996, + "token_usage": { + "input_tokens": 131884, + "output_tokens": 4949, + "total_tokens": 136833, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/dvdrental__film_inventory_management.json b/model_results/claude-sonnet-4-5/dvdrental__film_inventory_management.json new file mode 100644 index 00000000..e20a2b01 --- /dev/null +++ b/model_results/claude-sonnet-4-5/dvdrental__film_inventory_management.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "dvdrental__film_inventory_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 103.67711782455444, + "token_usage": { + "input_tokens": 441942, + "output_tokens": 3986, + "total_tokens": 445928, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 106.06457614898682, + "token_usage": { + "input_tokens": 252654, + "output_tokens": 4305, + "total_tokens": 256959, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 99.84581923484802, + "token_usage": { + "input_tokens": 179284, + "output_tokens": 3618, + "total_tokens": 182902, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 147.76059675216675, + "token_usage": { + "input_tokens": 512847, + "output_tokens": 4572, + "total_tokens": 517419, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/easyr1__advanced_branch_strategy.json b/model_results/claude-sonnet-4-5/easyr1__advanced_branch_strategy.json new file mode 100644 index 00000000..28cdb6f3 --- /dev/null +++ b/model_results/claude-sonnet-4-5/easyr1__advanced_branch_strategy.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "easyr1__advanced_branch_strategy", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 94.20567393302917, + "token_usage": { + "input_tokens": 243093, + "output_tokens": 2513, + "total_tokens": 245606, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 85.33322143554688, + "token_usage": { + "input_tokens": 258074, + "output_tokens": 2653, + "total_tokens": 260727, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 69.08874940872192, + "token_usage": { + "input_tokens": 163053, + "output_tokens": 2039, + "total_tokens": 165092, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 65.48941016197205, + "token_usage": { + "input_tokens": 180247, + "output_tokens": 1960, + "total_tokens": 182207, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/easyr1__config_parameter_audit.json b/model_results/claude-sonnet-4-5/easyr1__config_parameter_audit.json new file mode 100644 index 00000000..a922039c --- /dev/null +++ b/model_results/claude-sonnet-4-5/easyr1__config_parameter_audit.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "easyr1__config_parameter_audit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 240.67952251434326, + "token_usage": { + "input_tokens": 1607883, + "output_tokens": 3981, + "total_tokens": 1611864, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 113.6022355556488, + "token_usage": { + "input_tokens": 1198253, + "output_tokens": 3211, + "total_tokens": 1201464, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 210.83103489875793, + "token_usage": { + "input_tokens": 926443, + "output_tokens": 3065, + "total_tokens": 929508, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 116.27509450912476, + "token_usage": { + "input_tokens": 920720, + "output_tokens": 2926, + "total_tokens": 923646, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/easyr1__performance_regression_investigation.json b/model_results/claude-sonnet-4-5/easyr1__performance_regression_investigation.json new file mode 100644 index 00000000..a1e423ea --- /dev/null +++ b/model_results/claude-sonnet-4-5/easyr1__performance_regression_investigation.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "easyr1__performance_regression_investigation", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 125.40516710281372, + "token_usage": { + "input_tokens": 259202, + "output_tokens": 4610, + "total_tokens": 263812, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 102.49504208564758, + "token_usage": { + "input_tokens": 261044, + "output_tokens": 4027, + "total_tokens": 265071, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 157.9911093711853, + "token_usage": { + "input_tokens": 420174, + "output_tokens": 6557, + "total_tokens": 426731, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 169.86899995803833, + "token_usage": { + "input_tokens": 425687, + "output_tokens": 6912, + "total_tokens": 432599, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/easyr1__qwen3_issue_management.json b/model_results/claude-sonnet-4-5/easyr1__qwen3_issue_management.json new file mode 100644 index 00000000..37990777 --- /dev/null +++ b/model_results/claude-sonnet-4-5/easyr1__qwen3_issue_management.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "easyr1__qwen3_issue_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 48.81993341445923, + "token_usage": { + "input_tokens": 136452, + "output_tokens": 1438, + "total_tokens": 137890, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 58.35715985298157, + "token_usage": { + "input_tokens": 172705, + "output_tokens": 1805, + "total_tokens": 174510, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 63.62937140464783, + "token_usage": { + "input_tokens": 239392, + "output_tokens": 1478, + "total_tokens": 240870, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 48.61015057563782, + "token_usage": { + "input_tokens": 145628, + "output_tokens": 1428, + "total_tokens": 147056, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/employees__employee_demographics_report.json b/model_results/claude-sonnet-4-5/employees__employee_demographics_report.json new file mode 100644 index 00000000..6ffd71ef --- /dev/null +++ b/model_results/claude-sonnet-4-5/employees__employee_demographics_report.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "employees__employee_demographics_report", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 110.05146908760071, + "token_usage": { + "input_tokens": 136728, + "output_tokens": 5736, + "total_tokens": 142464, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 132.65369296073914, + "token_usage": { + "input_tokens": 188251, + "output_tokens": 5395, + "total_tokens": 193646, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 135.7118330001831, + "token_usage": { + "input_tokens": 209718, + "output_tokens": 6943, + "total_tokens": 216661, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 182.46520376205444, + "token_usage": { + "input_tokens": 277131, + "output_tokens": 8321, + "total_tokens": 285452, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/employees__employee_performance_analysis.json b/model_results/claude-sonnet-4-5/employees__employee_performance_analysis.json new file mode 100644 index 00000000..7ed80a67 --- /dev/null +++ b/model_results/claude-sonnet-4-5/employees__employee_performance_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "employees__employee_performance_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 349.44184613227844, + "token_usage": { + "input_tokens": 309034, + "output_tokens": 8445, + "total_tokens": 317479, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 118.50761294364929, + "token_usage": { + "input_tokens": 244312, + "output_tokens": 6021, + "total_tokens": 250333, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 153.22221660614014, + "token_usage": { + "input_tokens": 373968, + "output_tokens": 7602, + "total_tokens": 381570, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 162.31071543693542, + "token_usage": { + "input_tokens": 223674, + "output_tokens": 6990, + "total_tokens": 230664, + "reasoning_tokens": 0 + }, + "turn_count": 16 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/employees__employee_project_tracking.json b/model_results/claude-sonnet-4-5/employees__employee_project_tracking.json new file mode 100644 index 00000000..ae766a50 --- /dev/null +++ b/model_results/claude-sonnet-4-5/employees__employee_project_tracking.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "employees__employee_project_tracking", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 195.40135407447815, + "token_usage": { + "input_tokens": 690485, + "output_tokens": 7091, + "total_tokens": 697576, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 152.72725367546082, + "token_usage": { + "input_tokens": 357027, + "output_tokens": 6806, + "total_tokens": 363833, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 165.87249898910522, + "token_usage": { + "input_tokens": 377787, + "output_tokens": 7469, + "total_tokens": 385256, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 147.441415309906, + "token_usage": { + "input_tokens": 265250, + "output_tokens": 7180, + "total_tokens": 272430, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/employees__employee_retention_analysis.json b/model_results/claude-sonnet-4-5/employees__employee_retention_analysis.json new file mode 100644 index 00000000..a077f634 --- /dev/null +++ b/model_results/claude-sonnet-4-5/employees__employee_retention_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "employees__employee_retention_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 145.07664322853088, + "token_usage": { + "input_tokens": 323530, + "output_tokens": 6771, + "total_tokens": 330301, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 116.00009608268738, + "token_usage": { + "input_tokens": 192720, + "output_tokens": 5286, + "total_tokens": 198006, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 170.12169933319092, + "token_usage": { + "input_tokens": 499051, + "output_tokens": 7672, + "total_tokens": 506723, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 161.5325903892517, + "token_usage": { + "input_tokens": 269092, + "output_tokens": 6949, + "total_tokens": 276041, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/employees__executive_dashboard_automation.json b/model_results/claude-sonnet-4-5/employees__executive_dashboard_automation.json new file mode 100644 index 00000000..8d742a40 --- /dev/null +++ b/model_results/claude-sonnet-4-5/employees__executive_dashboard_automation.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "employees__executive_dashboard_automation", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 183.95886039733887, + "token_usage": { + "input_tokens": 532213, + "output_tokens": 6843, + "total_tokens": 539056, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 189.57184600830078, + "token_usage": { + "input_tokens": 593971, + "output_tokens": 7876, + "total_tokens": 601847, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 341.2478756904602, + "token_usage": { + "input_tokens": 671135, + "output_tokens": 12626, + "total_tokens": 683761, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 230.7133400440216, + "token_usage": { + "input_tokens": 492559, + "output_tokens": 8415, + "total_tokens": 500974, + "reasoning_tokens": 0 + }, + "turn_count": 32 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/employees__management_structure_analysis.json b/model_results/claude-sonnet-4-5/employees__management_structure_analysis.json new file mode 100644 index 00000000..eb01ef9f --- /dev/null +++ b/model_results/claude-sonnet-4-5/employees__management_structure_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "employees__management_structure_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 312.3840157985687, + "token_usage": { + "input_tokens": 292459, + "output_tokens": 6350, + "total_tokens": 298809, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 148.8743360042572, + "token_usage": { + "input_tokens": 360174, + "output_tokens": 7572, + "total_tokens": 367746, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 170.11345911026, + "token_usage": { + "input_tokens": 446019, + "output_tokens": 8377, + "total_tokens": 454396, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 118.14211320877075, + "token_usage": { + "input_tokens": 240100, + "output_tokens": 5535, + "total_tokens": 245635, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/eval_web__cloudflare_turnstile_challenge.json b/model_results/claude-sonnet-4-5/eval_web__cloudflare_turnstile_challenge.json new file mode 100644 index 00000000..c689cf0b --- /dev/null +++ b/model_results/claude-sonnet-4-5/eval_web__cloudflare_turnstile_challenge.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "eval_web__cloudflare_turnstile_challenge", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 253.90698075294495, + "token_usage": { + "input_tokens": 1753819, + "output_tokens": 4654, + "total_tokens": 1758473, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 570.5327370166779, + "token_usage": { + "input_tokens": 2558253, + "output_tokens": 15098, + "total_tokens": 2573351, + "reasoning_tokens": 0 + }, + "turn_count": 64 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 273.87959909439087, + "token_usage": { + "input_tokens": 1868567, + "output_tokens": 7582, + "total_tokens": 1876149, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 323.2769525051117, + "token_usage": { + "input_tokens": 1430075, + "output_tokens": 8372, + "total_tokens": 1438447, + "reasoning_tokens": 0 + }, + "turn_count": 47 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/eval_web__extraction_table.json b/model_results/claude-sonnet-4-5/eval_web__extraction_table.json new file mode 100644 index 00000000..a27b9df1 --- /dev/null +++ b/model_results/claude-sonnet-4-5/eval_web__extraction_table.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "eval_web__extraction_table", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 80.66271924972534, + "token_usage": { + "input_tokens": 108190, + "output_tokens": 3274, + "total_tokens": 111464, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 72.62936973571777, + "token_usage": { + "input_tokens": 100572, + "output_tokens": 3034, + "total_tokens": 103606, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 52.02479100227356, + "token_usage": { + "input_tokens": 24904, + "output_tokens": 2321, + "total_tokens": 27225, + "reasoning_tokens": 0 + }, + "turn_count": 3 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 79.66497540473938, + "token_usage": { + "input_tokens": 111918, + "output_tokens": 3491, + "total_tokens": 115409, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/file_context__duplicates_searching.json b/model_results/claude-sonnet-4-5/file_context__duplicates_searching.json new file mode 100644 index 00000000..15908303 --- /dev/null +++ b/model_results/claude-sonnet-4-5/file_context__duplicates_searching.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "file_context__duplicates_searching", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 89.57424807548523, + "token_usage": { + "input_tokens": 44023, + "output_tokens": 3856, + "total_tokens": 47879, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 54.63117861747742, + "token_usage": { + "input_tokens": 44273, + "output_tokens": 3853, + "total_tokens": 48126, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 44.39495873451233, + "token_usage": { + "input_tokens": 40743, + "output_tokens": 3046, + "total_tokens": 43789, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 53.31755542755127, + "token_usage": { + "input_tokens": 49810, + "output_tokens": 3916, + "total_tokens": 53726, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/file_context__file_merging.json b/model_results/claude-sonnet-4-5/file_context__file_merging.json new file mode 100644 index 00000000..266355f9 --- /dev/null +++ b/model_results/claude-sonnet-4-5/file_context__file_merging.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "file_context__file_merging", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 48.37791299819946, + "token_usage": { + "input_tokens": 27646, + "output_tokens": 1918, + "total_tokens": 29564, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 36.249232053756714, + "token_usage": { + "input_tokens": 28563, + "output_tokens": 1909, + "total_tokens": 30472, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 35.9812114238739, + "token_usage": { + "input_tokens": 30825, + "output_tokens": 1838, + "total_tokens": 32663, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 36.04903721809387, + "token_usage": { + "input_tokens": 28391, + "output_tokens": 1944, + "total_tokens": 30335, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/file_context__file_splitting.json b/model_results/claude-sonnet-4-5/file_context__file_splitting.json new file mode 100644 index 00000000..73ca60b9 --- /dev/null +++ b/model_results/claude-sonnet-4-5/file_context__file_splitting.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "file_context__file_splitting", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 779.1507678031921, + "token_usage": { + "input_tokens": 1813602, + "output_tokens": 25881, + "total_tokens": 1839483, + "reasoning_tokens": 0 + }, + "turn_count": 59 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 220.82219648361206, + "token_usage": { + "input_tokens": 33861, + "output_tokens": 2069, + "total_tokens": 35930, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 35.50087571144104, + "token_usage": { + "input_tokens": 27596, + "output_tokens": 1922, + "total_tokens": 29518, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 34.65121340751648, + "token_usage": { + "input_tokens": 27477, + "output_tokens": 1877, + "total_tokens": 29354, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/file_context__pattern_matching.json b/model_results/claude-sonnet-4-5/file_context__pattern_matching.json new file mode 100644 index 00000000..b3cdb198 --- /dev/null +++ b/model_results/claude-sonnet-4-5/file_context__pattern_matching.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "file_context__pattern_matching", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 58.629934549331665, + "token_usage": { + "input_tokens": 36762, + "output_tokens": 2859, + "total_tokens": 39621, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 48.12376308441162, + "token_usage": { + "input_tokens": 30432, + "output_tokens": 2844, + "total_tokens": 33276, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 40.02613186836243, + "token_usage": { + "input_tokens": 29981, + "output_tokens": 2330, + "total_tokens": 32311, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 40.59089994430542, + "token_usage": { + "input_tokens": 29908, + "output_tokens": 2282, + "total_tokens": 32190, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/file_context__uppercase.json b/model_results/claude-sonnet-4-5/file_context__uppercase.json new file mode 100644 index 00000000..b7706233 --- /dev/null +++ b/model_results/claude-sonnet-4-5/file_context__uppercase.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "file_context__uppercase", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 64.87658524513245, + "token_usage": { + "input_tokens": 47873, + "output_tokens": 3281, + "total_tokens": 51154, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 53.93935751914978, + "token_usage": { + "input_tokens": 48465, + "output_tokens": 3199, + "total_tokens": 51664, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 55.271063566207886, + "token_usage": { + "input_tokens": 57162, + "output_tokens": 3278, + "total_tokens": 60440, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 52.23234415054321, + "token_usage": { + "input_tokens": 48598, + "output_tokens": 3276, + "total_tokens": 51874, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/file_property__size_classification.json b/model_results/claude-sonnet-4-5/file_property__size_classification.json new file mode 100644 index 00000000..ac44813c --- /dev/null +++ b/model_results/claude-sonnet-4-5/file_property__size_classification.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "file_property__size_classification", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 104.95841813087463, + "token_usage": { + "input_tokens": 50594, + "output_tokens": 2909, + "total_tokens": 53503, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 42.95197820663452, + "token_usage": { + "input_tokens": 37903, + "output_tokens": 2914, + "total_tokens": 40817, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 48.14620399475098, + "token_usage": { + "input_tokens": 48099, + "output_tokens": 3176, + "total_tokens": 51275, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 41.206560373306274, + "token_usage": { + "input_tokens": 38015, + "output_tokens": 2944, + "total_tokens": 40959, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/file_property__time_classification.json b/model_results/claude-sonnet-4-5/file_property__time_classification.json new file mode 100644 index 00000000..8c51fdcf --- /dev/null +++ b/model_results/claude-sonnet-4-5/file_property__time_classification.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "file_property__time_classification", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 74.05369782447815, + "token_usage": { + "input_tokens": 80923, + "output_tokens": 3915, + "total_tokens": 84838, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 70.1641845703125, + "token_usage": { + "input_tokens": 68059, + "output_tokens": 3158, + "total_tokens": 71217, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 52.32022547721863, + "token_usage": { + "input_tokens": 59222, + "output_tokens": 3045, + "total_tokens": 62267, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 63.27241516113281, + "token_usage": { + "input_tokens": 75572, + "output_tokens": 3790, + "total_tokens": 79362, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/folder_structure__structure_analysis.json b/model_results/claude-sonnet-4-5/folder_structure__structure_analysis.json new file mode 100644 index 00000000..f6727382 --- /dev/null +++ b/model_results/claude-sonnet-4-5/folder_structure__structure_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "folder_structure__structure_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 221.27812266349792, + "token_usage": { + "input_tokens": 260402, + "output_tokens": 13428, + "total_tokens": 273830, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 337.2250006198883, + "token_usage": { + "input_tokens": 564713, + "output_tokens": 21575, + "total_tokens": 586288, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 280.85084867477417, + "token_usage": { + "input_tokens": 818881, + "output_tokens": 16340, + "total_tokens": 835221, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 221.56634426116943, + "token_usage": { + "input_tokens": 537992, + "output_tokens": 12023, + "total_tokens": 550015, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/folder_structure__structure_mirror.json b/model_results/claude-sonnet-4-5/folder_structure__structure_mirror.json new file mode 100644 index 00000000..7510b8cc --- /dev/null +++ b/model_results/claude-sonnet-4-5/folder_structure__structure_mirror.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "folder_structure__structure_mirror", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 111.14954137802124, + "token_usage": { + "input_tokens": 231693, + "output_tokens": 5858, + "total_tokens": 237551, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 230.31520056724548, + "token_usage": { + "input_tokens": 355034, + "output_tokens": 13517, + "total_tokens": 368551, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 73.10000491142273, + "token_usage": { + "input_tokens": 88445, + "output_tokens": 4797, + "total_tokens": 93242, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 162.5775134563446, + "token_usage": { + "input_tokens": 342432, + "output_tokens": 11913, + "total_tokens": 354345, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/harmony__fix_conflict.json b/model_results/claude-sonnet-4-5/harmony__fix_conflict.json new file mode 100644 index 00000000..e230189f --- /dev/null +++ b/model_results/claude-sonnet-4-5/harmony__fix_conflict.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "harmony__fix_conflict", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 271.94125032424927, + "token_usage": { + "input_tokens": 3238218, + "output_tokens": 6750, + "total_tokens": 3244968, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 231.37410593032837, + "token_usage": { + "input_tokens": 2520756, + "output_tokens": 6402, + "total_tokens": 2527158, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 381.10024642944336, + "token_usage": { + "input_tokens": 3948832, + "output_tokens": 7663, + "total_tokens": 3956495, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 233.51602387428284, + "token_usage": { + "input_tokens": 2641556, + "output_tokens": 5286, + "total_tokens": 2646842, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/harmony__issue_pr_commit_workflow.json b/model_results/claude-sonnet-4-5/harmony__issue_pr_commit_workflow.json new file mode 100644 index 00000000..870a2d1d --- /dev/null +++ b/model_results/claude-sonnet-4-5/harmony__issue_pr_commit_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "harmony__issue_pr_commit_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 95.93805837631226, + "token_usage": { + "input_tokens": 219281, + "output_tokens": 3445, + "total_tokens": 222726, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 93.80944776535034, + "token_usage": { + "input_tokens": 218845, + "output_tokens": 3301, + "total_tokens": 222146, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 98.31169533729553, + "token_usage": { + "input_tokens": 209796, + "output_tokens": 3487, + "total_tokens": 213283, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 119.56449246406555, + "token_usage": { + "input_tokens": 306150, + "output_tokens": 3870, + "total_tokens": 310020, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/harmony__issue_tagging_pr_closure.json b/model_results/claude-sonnet-4-5/harmony__issue_tagging_pr_closure.json new file mode 100644 index 00000000..5067460f --- /dev/null +++ b/model_results/claude-sonnet-4-5/harmony__issue_tagging_pr_closure.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "harmony__issue_tagging_pr_closure", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 180.46537899971008, + "token_usage": { + "input_tokens": 367451, + "output_tokens": 6002, + "total_tokens": 373453, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 175.22188591957092, + "token_usage": { + "input_tokens": 338439, + "output_tokens": 6445, + "total_tokens": 344884, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 151.25655937194824, + "token_usage": { + "input_tokens": 308568, + "output_tokens": 4979, + "total_tokens": 313547, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 155.67875576019287, + "token_usage": { + "input_tokens": 340027, + "output_tokens": 5143, + "total_tokens": 345170, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/harmony__multi_branch_commit_aggregation.json b/model_results/claude-sonnet-4-5/harmony__multi_branch_commit_aggregation.json new file mode 100644 index 00000000..219c3cf4 --- /dev/null +++ b/model_results/claude-sonnet-4-5/harmony__multi_branch_commit_aggregation.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "harmony__multi_branch_commit_aggregation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 146.04537844657898, + "token_usage": { + "input_tokens": 372609, + "output_tokens": 5671, + "total_tokens": 378280, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 96.12090182304382, + "token_usage": { + "input_tokens": 202555, + "output_tokens": 4053, + "total_tokens": 206608, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 136.93605017662048, + "token_usage": { + "input_tokens": 356151, + "output_tokens": 5470, + "total_tokens": 361621, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 166.02852249145508, + "token_usage": { + "input_tokens": 384563, + "output_tokens": 6307, + "total_tokens": 390870, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/harmony__release_management_workflow.json b/model_results/claude-sonnet-4-5/harmony__release_management_workflow.json new file mode 100644 index 00000000..5b621ba2 --- /dev/null +++ b/model_results/claude-sonnet-4-5/harmony__release_management_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "harmony__release_management_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 204.74576830863953, + "token_usage": { + "input_tokens": 1099725, + "output_tokens": 9419, + "total_tokens": 1109144, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 242.64568901062012, + "token_usage": { + "input_tokens": 1669027, + "output_tokens": 10407, + "total_tokens": 1679434, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 263.83993458747864, + "token_usage": { + "input_tokens": 1351392, + "output_tokens": 9899, + "total_tokens": 1361291, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 250.6093246936798, + "token_usage": { + "input_tokens": 1382339, + "output_tokens": 11189, + "total_tokens": 1393528, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/it_trouble_shooting_hub__asset_retirement_migration.json b/model_results/claude-sonnet-4-5/it_trouble_shooting_hub__asset_retirement_migration.json new file mode 100644 index 00000000..13aeb3fd --- /dev/null +++ b/model_results/claude-sonnet-4-5/it_trouble_shooting_hub__asset_retirement_migration.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "it_trouble_shooting_hub__asset_retirement_migration", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 117.95915627479553, + "token_usage": { + "input_tokens": 333402, + "output_tokens": 3060, + "total_tokens": 336462, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 239.47261881828308, + "token_usage": { + "input_tokens": 2312265, + "output_tokens": 6560, + "total_tokens": 2318825, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 185.89870953559875, + "token_usage": { + "input_tokens": 1846951, + "output_tokens": 4330, + "total_tokens": 1851281, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 223.02784872055054, + "token_usage": { + "input_tokens": 1209119, + "output_tokens": 5113, + "total_tokens": 1214232, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/it_trouble_shooting_hub__security_audit_ticket.json b/model_results/claude-sonnet-4-5/it_trouble_shooting_hub__security_audit_ticket.json new file mode 100644 index 00000000..a873b45e --- /dev/null +++ b/model_results/claude-sonnet-4-5/it_trouble_shooting_hub__security_audit_ticket.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "it_trouble_shooting_hub__security_audit_ticket", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 145.1133677959442, + "token_usage": { + "input_tokens": 418782, + "output_tokens": 2625, + "total_tokens": 421407, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 249.26999831199646, + "token_usage": { + "input_tokens": 1011424, + "output_tokens": 7724, + "total_tokens": 1019148, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 83.56927943229675, + "token_usage": { + "input_tokens": 402139, + "output_tokens": 2662, + "total_tokens": 404801, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 191.2257583141327, + "token_usage": { + "input_tokens": 811191, + "output_tokens": 4054, + "total_tokens": 815245, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/it_trouble_shooting_hub__verification_expired_update.json b/model_results/claude-sonnet-4-5/it_trouble_shooting_hub__verification_expired_update.json new file mode 100644 index 00000000..5c22bbe6 --- /dev/null +++ b/model_results/claude-sonnet-4-5/it_trouble_shooting_hub__verification_expired_update.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "it_trouble_shooting_hub__verification_expired_update", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 366.1456768512726, + "token_usage": { + "input_tokens": 1450904, + "output_tokens": 8911, + "total_tokens": 1459815, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 187.86872124671936, + "token_usage": { + "input_tokens": 1292817, + "output_tokens": 5843, + "total_tokens": 1298660, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 250.73846244812012, + "token_usage": { + "input_tokens": 1249920, + "output_tokens": 4355, + "total_tokens": 1254275, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 218.2809386253357, + "token_usage": { + "input_tokens": 1035185, + "output_tokens": 5107, + "total_tokens": 1040292, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/japan_travel_planner__daily_itinerary_overview.json b/model_results/claude-sonnet-4-5/japan_travel_planner__daily_itinerary_overview.json new file mode 100644 index 00000000..71bef5e1 --- /dev/null +++ b/model_results/claude-sonnet-4-5/japan_travel_planner__daily_itinerary_overview.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "japan_travel_planner__daily_itinerary_overview", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 81.93832159042358, + "token_usage": { + "input_tokens": 281329, + "output_tokens": 4319, + "total_tokens": 285648, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 73.66770553588867, + "token_usage": { + "input_tokens": 269195, + "output_tokens": 4516, + "total_tokens": 273711, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 78.67584681510925, + "token_usage": { + "input_tokens": 326669, + "output_tokens": 4199, + "total_tokens": 330868, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 73.6284589767456, + "token_usage": { + "input_tokens": 283316, + "output_tokens": 4912, + "total_tokens": 288228, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/japan_travel_planner__packing_progress_summary.json b/model_results/claude-sonnet-4-5/japan_travel_planner__packing_progress_summary.json new file mode 100644 index 00000000..f067ba93 --- /dev/null +++ b/model_results/claude-sonnet-4-5/japan_travel_planner__packing_progress_summary.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "japan_travel_planner__packing_progress_summary", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 116.83711194992065, + "token_usage": { + "input_tokens": 436667, + "output_tokens": 3514, + "total_tokens": 440181, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 126.43457913398743, + "token_usage": { + "input_tokens": 889598, + "output_tokens": 4321, + "total_tokens": 893919, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 117.8135359287262, + "token_usage": { + "input_tokens": 483602, + "output_tokens": 3648, + "total_tokens": 487250, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 162.7522430419922, + "token_usage": { + "input_tokens": 813250, + "output_tokens": 5227, + "total_tokens": 818477, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/japan_travel_planner__remove_osaka_itinerary.json b/model_results/claude-sonnet-4-5/japan_travel_planner__remove_osaka_itinerary.json new file mode 100644 index 00000000..bae3a66f --- /dev/null +++ b/model_results/claude-sonnet-4-5/japan_travel_planner__remove_osaka_itinerary.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "japan_travel_planner__remove_osaka_itinerary", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 44.59681296348572, + "token_usage": { + "input_tokens": 201381, + "output_tokens": 1376, + "total_tokens": 202757, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 36.224655628204346, + "token_usage": { + "input_tokens": 179392, + "output_tokens": 1099, + "total_tokens": 180491, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 180.99512124061584, + "token_usage": { + "input_tokens": 470201, + "output_tokens": 1458, + "total_tokens": 471659, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 42.02324724197388, + "token_usage": { + "input_tokens": 206275, + "output_tokens": 1094, + "total_tokens": 207369, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/japan_travel_planner__restaurant_expenses_sync.json b/model_results/claude-sonnet-4-5/japan_travel_planner__restaurant_expenses_sync.json new file mode 100644 index 00000000..755cc96e --- /dev/null +++ b/model_results/claude-sonnet-4-5/japan_travel_planner__restaurant_expenses_sync.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "japan_travel_planner__restaurant_expenses_sync", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 111.48391270637512, + "token_usage": { + "input_tokens": 286204, + "output_tokens": 3778, + "total_tokens": 289982, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 68.60682034492493, + "token_usage": { + "input_tokens": 158082, + "output_tokens": 2042, + "total_tokens": 160124, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 114.06821250915527, + "token_usage": { + "input_tokens": 296315, + "output_tokens": 4247, + "total_tokens": 300562, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 81.31473469734192, + "token_usage": { + "input_tokens": 124759, + "output_tokens": 2757, + "total_tokens": 127516, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/legal_document__dispute_review.json b/model_results/claude-sonnet-4-5/legal_document__dispute_review.json new file mode 100644 index 00000000..dfc80ac3 --- /dev/null +++ b/model_results/claude-sonnet-4-5/legal_document__dispute_review.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "legal_document__dispute_review", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 45.63363075256348, + "token_usage": { + "input_tokens": 154615, + "output_tokens": 1700, + "total_tokens": 156315, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 39.39472794532776, + "token_usage": { + "input_tokens": 154549, + "output_tokens": 1542, + "total_tokens": 156091, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 43.42288160324097, + "token_usage": { + "input_tokens": 154625, + "output_tokens": 1639, + "total_tokens": 156264, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 52.20779824256897, + "token_usage": { + "input_tokens": 154136, + "output_tokens": 1641, + "total_tokens": 155777, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/legal_document__individual_comments.json b/model_results/claude-sonnet-4-5/legal_document__individual_comments.json new file mode 100644 index 00000000..c1d6a860 --- /dev/null +++ b/model_results/claude-sonnet-4-5/legal_document__individual_comments.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "legal_document__individual_comments", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 77.71043944358826, + "token_usage": { + "input_tokens": 195433, + "output_tokens": 2195, + "total_tokens": 197628, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 51.940072774887085, + "token_usage": { + "input_tokens": 199357, + "output_tokens": 2166, + "total_tokens": 201523, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 50.997467041015625, + "token_usage": { + "input_tokens": 195373, + "output_tokens": 2296, + "total_tokens": 197669, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 40.61216926574707, + "token_usage": { + "input_tokens": 198656, + "output_tokens": 1591, + "total_tokens": 200247, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/legal_document__solution_tracing.json b/model_results/claude-sonnet-4-5/legal_document__solution_tracing.json new file mode 100644 index 00000000..c3b88fc8 --- /dev/null +++ b/model_results/claude-sonnet-4-5/legal_document__solution_tracing.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "legal_document__solution_tracing", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 55.318326234817505, + "token_usage": { + "input_tokens": 286493, + "output_tokens": 1998, + "total_tokens": 288491, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 50.05606198310852, + "token_usage": { + "input_tokens": 286573, + "output_tokens": 1895, + "total_tokens": 288468, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 49.31079387664795, + "token_usage": { + "input_tokens": 286475, + "output_tokens": 1794, + "total_tokens": 288269, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 50.229806423187256, + "token_usage": { + "input_tokens": 286461, + "output_tokens": 1827, + "total_tokens": 288288, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/lego__consistency_enforcement.json b/model_results/claude-sonnet-4-5/lego__consistency_enforcement.json new file mode 100644 index 00000000..8415f363 --- /dev/null +++ b/model_results/claude-sonnet-4-5/lego__consistency_enforcement.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "lego__consistency_enforcement", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 203.31866574287415, + "token_usage": { + "input_tokens": 336058, + "output_tokens": 7078, + "total_tokens": 343136, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 212.3275921344757, + "token_usage": { + "input_tokens": 1649469, + "output_tokens": 7648, + "total_tokens": 1657117, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 176.8132345676422, + "token_usage": { + "input_tokens": 518130, + "output_tokens": 7826, + "total_tokens": 525956, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 214.29601955413818, + "token_usage": { + "input_tokens": 1491130, + "output_tokens": 8038, + "total_tokens": 1499168, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/lego__database_security_policies.json b/model_results/claude-sonnet-4-5/lego__database_security_policies.json new file mode 100644 index 00000000..3b478960 --- /dev/null +++ b/model_results/claude-sonnet-4-5/lego__database_security_policies.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "lego__database_security_policies", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 403.9742548465729, + "token_usage": { + "input_tokens": 714649, + "output_tokens": 13742, + "total_tokens": 728391, + "reasoning_tokens": 0 + }, + "turn_count": 43 + }, + "run-2": { + "success": true, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 207335 tokens \\u003e 200000 maximum (tid: 2025093019124855593274627890108)\"},\"type\":\"error\"}", + "execution_time": 108.09306788444519, + "token_usage": { + "input_tokens": 196827, + "output_tokens": 3976, + "total_tokens": 200803, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": true, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200038 tokens \\u003e 200000 maximum (tid: 2025100101360313161439216047926)\"},\"type\":\"error\"}", + "execution_time": 111.85084342956543, + "token_usage": { + "input_tokens": 231728, + "output_tokens": 3763, + "total_tokens": 235491, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 220.7334008216858, + "token_usage": { + "input_tokens": 370627, + "output_tokens": 11738, + "total_tokens": 382365, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/lego__transactional_inventory_transfer.json b/model_results/claude-sonnet-4-5/lego__transactional_inventory_transfer.json new file mode 100644 index 00000000..9feef35f --- /dev/null +++ b/model_results/claude-sonnet-4-5/lego__transactional_inventory_transfer.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "lego__transactional_inventory_transfer", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 616.4718172550201, + "token_usage": { + "input_tokens": 1179003, + "output_tokens": 18991, + "total_tokens": 1197994, + "reasoning_tokens": 0 + }, + "turn_count": 51 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 376.94758009910583, + "token_usage": { + "input_tokens": 1150560, + "output_tokens": 17002, + "total_tokens": 1167562, + "reasoning_tokens": 0 + }, + "turn_count": 54 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 283.57749247550964, + "token_usage": { + "input_tokens": 841284, + "output_tokens": 12400, + "total_tokens": 853684, + "reasoning_tokens": 0 + }, + "turn_count": 41 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 551.8978893756866, + "token_usage": { + "input_tokens": 1863746, + "output_tokens": 25469, + "total_tokens": 1889215, + "reasoning_tokens": 0 + }, + "turn_count": 64 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/mcpmark-cicd__deployment_status_workflow.json b/model_results/claude-sonnet-4-5/mcpmark-cicd__deployment_status_workflow.json new file mode 100644 index 00000000..169a0888 --- /dev/null +++ b/model_results/claude-sonnet-4-5/mcpmark-cicd__deployment_status_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "mcpmark-cicd__deployment_status_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 523.5952243804932, + "token_usage": { + "input_tokens": 876833, + "output_tokens": 32914, + "total_tokens": 909747, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 319.86090779304504, + "token_usage": { + "input_tokens": 882718, + "output_tokens": 16761, + "total_tokens": 899479, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 892.861257314682, + "token_usage": { + "input_tokens": 2449736, + "output_tokens": 49222, + "total_tokens": 2498958, + "reasoning_tokens": 0 + }, + "turn_count": 82 + }, + "run-4": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 827.4493978023529, + "token_usage": { + "input_tokens": 3133840, + "output_tokens": 43170, + "total_tokens": 3177010, + "reasoning_tokens": 0 + }, + "turn_count": 100 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/mcpmark-cicd__issue_management_workflow.json b/model_results/claude-sonnet-4-5/mcpmark-cicd__issue_management_workflow.json new file mode 100644 index 00000000..e50c8c73 --- /dev/null +++ b/model_results/claude-sonnet-4-5/mcpmark-cicd__issue_management_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "mcpmark-cicd__issue_management_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 353.6409010887146, + "token_usage": { + "input_tokens": 648047, + "output_tokens": 21611, + "total_tokens": 669658, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 567.3189604282379, + "token_usage": { + "input_tokens": 1427391, + "output_tokens": 17732, + "total_tokens": 1445123, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 504.7355761528015, + "token_usage": { + "input_tokens": 771229, + "output_tokens": 33138, + "total_tokens": 804367, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 556.5370657444, + "token_usage": { + "input_tokens": 1058191, + "output_tokens": 34557, + "total_tokens": 1092748, + "reasoning_tokens": 0 + }, + "turn_count": 28 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/mcpmark-cicd__linting_ci_workflow.json b/model_results/claude-sonnet-4-5/mcpmark-cicd__linting_ci_workflow.json new file mode 100644 index 00000000..bf4a4689 --- /dev/null +++ b/model_results/claude-sonnet-4-5/mcpmark-cicd__linting_ci_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "mcpmark-cicd__linting_ci_workflow", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 80.46243286132812, + "token_usage": { + "input_tokens": 196962, + "output_tokens": 3144, + "total_tokens": 200106, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 73.94100975990295, + "token_usage": { + "input_tokens": 174836, + "output_tokens": 2907, + "total_tokens": 177743, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 139.19495487213135, + "token_usage": { + "input_tokens": 209881, + "output_tokens": 2957, + "total_tokens": 212838, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 76.99708795547485, + "token_usage": { + "input_tokens": 184479, + "output_tokens": 2794, + "total_tokens": 187273, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/mcpmark-cicd__pr_automation_workflow.json b/model_results/claude-sonnet-4-5/mcpmark-cicd__pr_automation_workflow.json new file mode 100644 index 00000000..6a978095 --- /dev/null +++ b/model_results/claude-sonnet-4-5/mcpmark-cicd__pr_automation_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "mcpmark-cicd__pr_automation_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 399.36375999450684, + "token_usage": { + "input_tokens": 889788, + "output_tokens": 24070, + "total_tokens": 913858, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 883.0230395793915, + "token_usage": { + "input_tokens": 3024248, + "output_tokens": 50971, + "total_tokens": 3075219, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 436.3579466342926, + "token_usage": { + "input_tokens": 952340, + "output_tokens": 28101, + "total_tokens": 980441, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 566.5929629802704, + "token_usage": { + "input_tokens": 644382, + "output_tokens": 35768, + "total_tokens": 680150, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/missing-semester__assign_contributor_labels.json b/model_results/claude-sonnet-4-5/missing-semester__assign_contributor_labels.json new file mode 100644 index 00000000..2573e4b5 --- /dev/null +++ b/model_results/claude-sonnet-4-5/missing-semester__assign_contributor_labels.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "missing-semester__assign_contributor_labels", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 108.81156086921692, + "token_usage": { + "input_tokens": 577979, + "output_tokens": 4005, + "total_tokens": 581984, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 69.24230194091797, + "token_usage": { + "input_tokens": 278352, + "output_tokens": 2642, + "total_tokens": 280994, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 97.96777606010437, + "token_usage": { + "input_tokens": 523025, + "output_tokens": 3554, + "total_tokens": 526579, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 83.05909609794617, + "token_usage": { + "input_tokens": 378019, + "output_tokens": 3102, + "total_tokens": 381121, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/missing-semester__find_legacy_name.json b/model_results/claude-sonnet-4-5/missing-semester__find_legacy_name.json new file mode 100644 index 00000000..db754384 --- /dev/null +++ b/model_results/claude-sonnet-4-5/missing-semester__find_legacy_name.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "missing-semester__find_legacy_name", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 98.60856485366821, + "token_usage": { + "input_tokens": 1275937, + "output_tokens": 1705, + "total_tokens": 1277642, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 109.07105731964111, + "token_usage": { + "input_tokens": 1349140, + "output_tokens": 2386, + "total_tokens": 1351526, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 153.1866273880005, + "token_usage": { + "input_tokens": 1860761, + "output_tokens": 2830, + "total_tokens": 1863591, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 101.16169309616089, + "token_usage": { + "input_tokens": 1345107, + "output_tokens": 1906, + "total_tokens": 1347013, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/missing-semester__find_salient_file.json b/model_results/claude-sonnet-4-5/missing-semester__find_salient_file.json new file mode 100644 index 00000000..f331c46b --- /dev/null +++ b/model_results/claude-sonnet-4-5/missing-semester__find_salient_file.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "github", + "task": "missing-semester__find_salient_file", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 372.2928993701935, + "token_usage": { + "input_tokens": 1879587, + "output_tokens": 11574, + "total_tokens": 1891161, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 139.31281876564026, + "token_usage": { + "input_tokens": 1059119, + "output_tokens": 3923, + "total_tokens": 1063042, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 306.26760482788086, + "token_usage": { + "input_tokens": 1449960, + "output_tokens": 9016, + "total_tokens": 1458976, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 389.31269216537476, + "token_usage": { + "input_tokens": 1915652, + "output_tokens": 12221, + "total_tokens": 1927873, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/online_resume__layout_adjustment.json b/model_results/claude-sonnet-4-5/online_resume__layout_adjustment.json new file mode 100644 index 00000000..f5cd2e2e --- /dev/null +++ b/model_results/claude-sonnet-4-5/online_resume__layout_adjustment.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "online_resume__layout_adjustment", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 148.5750207901001, + "token_usage": { + "input_tokens": 552532, + "output_tokens": 4933, + "total_tokens": 557465, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 157.71774888038635, + "token_usage": { + "input_tokens": 683061, + "output_tokens": 5658, + "total_tokens": 688719, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 190.4845688343048, + "token_usage": { + "input_tokens": 799547, + "output_tokens": 5602, + "total_tokens": 805149, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 165.11123919487, + "token_usage": { + "input_tokens": 595091, + "output_tokens": 4789, + "total_tokens": 599880, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/online_resume__projects_section_update.json b/model_results/claude-sonnet-4-5/online_resume__projects_section_update.json new file mode 100644 index 00000000..187a8419 --- /dev/null +++ b/model_results/claude-sonnet-4-5/online_resume__projects_section_update.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "online_resume__projects_section_update", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 82.00306606292725, + "token_usage": { + "input_tokens": 308566, + "output_tokens": 2697, + "total_tokens": 311263, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 119.67876672744751, + "token_usage": { + "input_tokens": 332267, + "output_tokens": 3807, + "total_tokens": 336074, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 194.65828442573547, + "token_usage": { + "input_tokens": 554726, + "output_tokens": 6195, + "total_tokens": 560921, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 212.98318338394165, + "token_usage": { + "input_tokens": 1334677, + "output_tokens": 4448, + "total_tokens": 1339125, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/online_resume__skills_development_tracker.json b/model_results/claude-sonnet-4-5/online_resume__skills_development_tracker.json new file mode 100644 index 00000000..39db2b34 --- /dev/null +++ b/model_results/claude-sonnet-4-5/online_resume__skills_development_tracker.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "online_resume__skills_development_tracker", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 259.40184354782104, + "token_usage": { + "input_tokens": 729375, + "output_tokens": 7270, + "total_tokens": 736645, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 118.78692293167114, + "token_usage": { + "input_tokens": 205288, + "output_tokens": 3679, + "total_tokens": 208967, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 134.834406375885, + "token_usage": { + "input_tokens": 527634, + "output_tokens": 4964, + "total_tokens": 532598, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 300.42046642303467, + "token_usage": { + "input_tokens": 1284658, + "output_tokens": 5675, + "total_tokens": 1290333, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/online_resume__work_history_addition.json b/model_results/claude-sonnet-4-5/online_resume__work_history_addition.json new file mode 100644 index 00000000..1df432ae --- /dev/null +++ b/model_results/claude-sonnet-4-5/online_resume__work_history_addition.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "online_resume__work_history_addition", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 112.69288372993469, + "token_usage": { + "input_tokens": 322254, + "output_tokens": 3549, + "total_tokens": 325803, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 130.8178744316101, + "token_usage": { + "input_tokens": 441689, + "output_tokens": 4362, + "total_tokens": 446051, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 102.33282542228699, + "token_usage": { + "input_tokens": 178051, + "output_tokens": 2601, + "total_tokens": 180652, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 118.62417793273926, + "token_usage": { + "input_tokens": 662244, + "output_tokens": 3312, + "total_tokens": 665556, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/papers__author_folders.json b/model_results/claude-sonnet-4-5/papers__author_folders.json new file mode 100644 index 00000000..7756ed36 --- /dev/null +++ b/model_results/claude-sonnet-4-5/papers__author_folders.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "papers__author_folders", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 212286 tokens \\u003e 200000 maximum (tid: 2025093008112950663022485543859)\"},\"type\":\"error\"}", + "execution_time": 123.42005062103271, + "token_usage": { + "input_tokens": 1058921, + "output_tokens": 4521, + "total_tokens": 1063442, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 225.0844805240631, + "token_usage": { + "input_tokens": 2154109, + "output_tokens": 10482, + "total_tokens": 2164591, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 211883 tokens \\u003e 200000 maximum (tid: 2025093009492196696539601709935)\"},\"type\":\"error\"}", + "execution_time": 54.932896852493286, + "token_usage": { + "input_tokens": 77098, + "output_tokens": 1142, + "total_tokens": 78240, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 212303 tokens \\u003e 200000 maximum (tid: 2025093010353288353095455414940)\"},\"type\":\"error\"}", + "execution_time": 34.484243869781494, + "token_usage": { + "input_tokens": 27922, + "output_tokens": 963, + "total_tokens": 28885, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/papers__find_math_paper.json b/model_results/claude-sonnet-4-5/papers__find_math_paper.json new file mode 100644 index 00000000..b35b9189 --- /dev/null +++ b/model_results/claude-sonnet-4-5/papers__find_math_paper.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "papers__find_math_paper", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 212599 tokens \\u003e 200000 maximum (tid: 2025093008124581919684350073652)\"},\"type\":\"error\"}", + "execution_time": 72.36091184616089, + "token_usage": { + "input_tokens": 725195, + "output_tokens": 1548, + "total_tokens": 726743, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 95.37920951843262, + "token_usage": { + "input_tokens": 767025, + "output_tokens": 3305, + "total_tokens": 770330, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 126.08392524719238, + "token_usage": { + "input_tokens": 1237899, + "output_tokens": 3179, + "total_tokens": 1241078, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 90.26646852493286, + "token_usage": { + "input_tokens": 741780, + "output_tokens": 2902, + "total_tokens": 744682, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/papers__organize_legacy_papers.json b/model_results/claude-sonnet-4-5/papers__organize_legacy_papers.json new file mode 100644 index 00000000..6d83cca4 --- /dev/null +++ b/model_results/claude-sonnet-4-5/papers__organize_legacy_papers.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "papers__organize_legacy_papers", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 212956 tokens \\u003e 200000 maximum (tid: 2025093008132258694300084726563)\"},\"type\":\"error\"}", + "execution_time": 33.34410619735718, + "token_usage": { + "input_tokens": 17860, + "output_tokens": 922, + "total_tokens": 18782, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 213356 tokens \\u003e 200000 maximum (tid: 2025093009074865911699300526349)\"},\"type\":\"error\"}", + "execution_time": 41.60207962989807, + "token_usage": { + "input_tokens": 21516, + "output_tokens": 919, + "total_tokens": 22435, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 212943 tokens \\u003e 200000 maximum (tid: 2025093009521293990005748006236)\"},\"type\":\"error\"}", + "execution_time": 40.0706422328949, + "token_usage": { + "input_tokens": 17903, + "output_tokens": 974, + "total_tokens": 18877, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 213354 tokens \\u003e 200000 maximum (tid: 2025093010373122994558277493562)\"},\"type\":\"error\"}", + "execution_time": 25.571825742721558, + "token_usage": { + "input_tokens": 10866, + "output_tokens": 852, + "total_tokens": 11718, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/python_roadmap__expert_level_lessons.json b/model_results/claude-sonnet-4-5/python_roadmap__expert_level_lessons.json new file mode 100644 index 00000000..5d466066 --- /dev/null +++ b/model_results/claude-sonnet-4-5/python_roadmap__expert_level_lessons.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "python_roadmap__expert_level_lessons", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 313.82976055145264, + "token_usage": { + "input_tokens": 1732854, + "output_tokens": 10110, + "total_tokens": 1742964, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 182.64713835716248, + "token_usage": { + "input_tokens": 910520, + "output_tokens": 6510, + "total_tokens": 917030, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 183.46198868751526, + "token_usage": { + "input_tokens": 1153274, + "output_tokens": 6006, + "total_tokens": 1159280, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 334.6447241306305, + "token_usage": { + "input_tokens": 1469449, + "output_tokens": 6879, + "total_tokens": 1476328, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/python_roadmap__learning_metrics_dashboard.json b/model_results/claude-sonnet-4-5/python_roadmap__learning_metrics_dashboard.json new file mode 100644 index 00000000..c3ba1054 --- /dev/null +++ b/model_results/claude-sonnet-4-5/python_roadmap__learning_metrics_dashboard.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "python_roadmap__learning_metrics_dashboard", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 165.26004815101624, + "token_usage": { + "input_tokens": 996104, + "output_tokens": 4752, + "total_tokens": 1000856, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 125.44312357902527, + "token_usage": { + "input_tokens": 702217, + "output_tokens": 3752, + "total_tokens": 705969, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 176.05701327323914, + "token_usage": { + "input_tokens": 1190124, + "output_tokens": 5361, + "total_tokens": 1195485, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 301.2488284111023, + "token_usage": { + "input_tokens": 890721, + "output_tokens": 7647, + "total_tokens": 898368, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/reddit__ai_data_analyst.json b/model_results/claude-sonnet-4-5/reddit__ai_data_analyst.json new file mode 100644 index 00000000..312bc903 --- /dev/null +++ b/model_results/claude-sonnet-4-5/reddit__ai_data_analyst.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "reddit__ai_data_analyst", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 202208 tokens \\u003e 200000 maximum (tid: 2025093017062826115448082297269)\"},\"type\":\"error\"}", + "execution_time": 122.39360022544861, + "token_usage": { + "input_tokens": 810885, + "output_tokens": 2543, + "total_tokens": 813428, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 208376 tokens \\u003e 200000 maximum (tid: 2025100103144066978161384663152)\"},\"type\":\"error\"}", + "execution_time": 175.28705477714539, + "token_usage": { + "input_tokens": 1430998, + "output_tokens": 3082, + "total_tokens": 1434080, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200958 tokens \\u003e 200000 maximum (tid: 2025100104361876637420905286286)\"},\"type\":\"error\"}", + "execution_time": 122.74756741523743, + "token_usage": { + "input_tokens": 739859, + "output_tokens": 2124, + "total_tokens": 741983, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 204619 tokens \\u003e 200000 maximum (tid: 2025100111024039322982561308244)\"},\"type\":\"error\"}", + "execution_time": 155.65950226783752, + "token_usage": { + "input_tokens": 1219293, + "output_tokens": 2811, + "total_tokens": 1222104, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/reddit__budget_europe_travel.json b/model_results/claude-sonnet-4-5/reddit__budget_europe_travel.json new file mode 100644 index 00000000..25b0a5c8 --- /dev/null +++ b/model_results/claude-sonnet-4-5/reddit__budget_europe_travel.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "reddit__budget_europe_travel", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 196.1325080394745, + "token_usage": { + "input_tokens": 1064397, + "output_tokens": 4494, + "total_tokens": 1068891, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 205.26203393936157, + "token_usage": { + "input_tokens": 888870, + "output_tokens": 4107, + "total_tokens": 892977, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 169.2699544429779, + "token_usage": { + "input_tokens": 811080, + "output_tokens": 4110, + "total_tokens": 815190, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 236.92507910728455, + "token_usage": { + "input_tokens": 1136974, + "output_tokens": 5089, + "total_tokens": 1142063, + "reasoning_tokens": 0 + }, + "turn_count": 36 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/reddit__buyitforlife_research.json b/model_results/claude-sonnet-4-5/reddit__buyitforlife_research.json new file mode 100644 index 00000000..1497bdb1 --- /dev/null +++ b/model_results/claude-sonnet-4-5/reddit__buyitforlife_research.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "reddit__buyitforlife_research", + "runs": { + "run-1": { + "success": true, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200608 tokens \\u003e 200000 maximum (tid: 2025093017135690131686640557860)\"},\"type\":\"error\"}", + "execution_time": 157.67092394828796, + "token_usage": { + "input_tokens": 1323550, + "output_tokens": 2816, + "total_tokens": 1326366, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 206209 tokens \\u003e 200000 maximum (tid: 2025100103215856062508834639915)\"},\"type\":\"error\"}", + "execution_time": 128.9041256904602, + "token_usage": { + "input_tokens": 607159, + "output_tokens": 2170, + "total_tokens": 609329, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 134.75087237358093, + "token_usage": { + "input_tokens": 1136386, + "output_tokens": 2832, + "total_tokens": 1139218, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 206236 tokens \\u003e 200000 maximum (tid: 2025100111100669782900763318858)\"},\"type\":\"error\"}", + "execution_time": 115.47299408912659, + "token_usage": { + "input_tokens": 625726, + "output_tokens": 1954, + "total_tokens": 627680, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/reddit__llm_research_summary.json b/model_results/claude-sonnet-4-5/reddit__llm_research_summary.json new file mode 100644 index 00000000..b66277ab --- /dev/null +++ b/model_results/claude-sonnet-4-5/reddit__llm_research_summary.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "reddit__llm_research_summary", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 201198 tokens \\u003e 200000 maximum (tid: 2025093017161371801536089254065)\"},\"type\":\"error\"}", + "execution_time": 96.99859285354614, + "token_usage": { + "input_tokens": 473634, + "output_tokens": 2536, + "total_tokens": 476170, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 166.61272382736206, + "token_usage": { + "input_tokens": 1492546, + "output_tokens": 4520, + "total_tokens": 1497066, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200903 tokens \\u003e 200000 maximum (tid: 2025100104464561942334456805258)\"},\"type\":\"error\"}", + "execution_time": 186.2016956806183, + "token_usage": { + "input_tokens": 1574490, + "output_tokens": 3844, + "total_tokens": 1578334, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 172.61223649978638, + "token_usage": { + "input_tokens": 1512269, + "output_tokens": 4259, + "total_tokens": 1516528, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/reddit__movie_reviewer_analysis.json b/model_results/claude-sonnet-4-5/reddit__movie_reviewer_analysis.json new file mode 100644 index 00000000..2b81c95a --- /dev/null +++ b/model_results/claude-sonnet-4-5/reddit__movie_reviewer_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "reddit__movie_reviewer_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 153.9569227695465, + "token_usage": { + "input_tokens": 753348, + "output_tokens": 4743, + "total_tokens": 758091, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 146.11267495155334, + "token_usage": { + "input_tokens": 764902, + "output_tokens": 4362, + "total_tokens": 769264, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 219.72306609153748, + "token_usage": { + "input_tokens": 1067686, + "output_tokens": 4512, + "total_tokens": 1072198, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 156.99191880226135, + "token_usage": { + "input_tokens": 947009, + "output_tokens": 3929, + "total_tokens": 950938, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/reddit__nba_statistics_analysis.json b/model_results/claude-sonnet-4-5/reddit__nba_statistics_analysis.json new file mode 100644 index 00000000..32639c85 --- /dev/null +++ b/model_results/claude-sonnet-4-5/reddit__nba_statistics_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "reddit__nba_statistics_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 184.2265748977661, + "token_usage": { + "input_tokens": 1749983, + "output_tokens": 4226, + "total_tokens": 1754209, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 205.5121591091156, + "token_usage": { + "input_tokens": 1649711, + "output_tokens": 4362, + "total_tokens": 1654073, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 213907 tokens \\u003e 200000 maximum (tid: 2025100109585427004089028603992)\"},\"type\":\"error\"}", + "execution_time": 262.53530740737915, + "token_usage": { + "input_tokens": 1455001, + "output_tokens": 3098, + "total_tokens": 1458099, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 198.60794973373413, + "token_usage": { + "input_tokens": 1853255, + "output_tokens": 3791, + "total_tokens": 1857046, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/reddit__routine_tracker_forum.json b/model_results/claude-sonnet-4-5/reddit__routine_tracker_forum.json new file mode 100644 index 00000000..39a316f8 --- /dev/null +++ b/model_results/claude-sonnet-4-5/reddit__routine_tracker_forum.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "reddit__routine_tracker_forum", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 127.6527099609375, + "token_usage": { + "input_tokens": 1385788, + "output_tokens": 2353, + "total_tokens": 1388141, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": true, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 209000 tokens \\u003e 200000 maximum (tid: 2025100103382013266947378383241)\"},\"type\":\"error\"}", + "execution_time": 274.5775909423828, + "token_usage": { + "input_tokens": 1424503, + "output_tokens": 2314, + "total_tokens": 1426817, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 239.11703038215637, + "token_usage": { + "input_tokens": 1535007, + "output_tokens": 3000, + "total_tokens": 1538007, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": true, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 208995 tokens \\u003e 200000 maximum (tid: 2025100111244513901080449091391)\"},\"type\":\"error\"}", + "execution_time": 170.82811069488525, + "token_usage": { + "input_tokens": 1610847, + "output_tokens": 2632, + "total_tokens": 1613479, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/security__rls_business_access.json b/model_results/claude-sonnet-4-5/security__rls_business_access.json new file mode 100644 index 00000000..5952fbfd --- /dev/null +++ b/model_results/claude-sonnet-4-5/security__rls_business_access.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "security__rls_business_access", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 230.73492789268494, + "token_usage": { + "input_tokens": 480372, + "output_tokens": 13967, + "total_tokens": 494339, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 207.54959177970886, + "token_usage": { + "input_tokens": 326409, + "output_tokens": 12702, + "total_tokens": 339111, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 182.66811847686768, + "token_usage": { + "input_tokens": 409390, + "output_tokens": 9802, + "total_tokens": 419192, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 171.18537974357605, + "token_usage": { + "input_tokens": 311090, + "output_tokens": 9247, + "total_tokens": 320337, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/security__user_permission_audit.json b/model_results/claude-sonnet-4-5/security__user_permission_audit.json new file mode 100644 index 00000000..e5e12d8d --- /dev/null +++ b/model_results/claude-sonnet-4-5/security__user_permission_audit.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "security__user_permission_audit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 102.74184060096741, + "token_usage": { + "input_tokens": 149227, + "output_tokens": 6250, + "total_tokens": 155477, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 114.31816673278809, + "token_usage": { + "input_tokens": 185891, + "output_tokens": 6043, + "total_tokens": 191934, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 164.20190405845642, + "token_usage": { + "input_tokens": 322751, + "output_tokens": 9400, + "total_tokens": 332151, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 135.47405791282654, + "token_usage": { + "input_tokens": 271906, + "output_tokens": 6009, + "total_tokens": 277915, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/self_assessment__faq_column_layout.json b/model_results/claude-sonnet-4-5/self_assessment__faq_column_layout.json new file mode 100644 index 00000000..04247c3a --- /dev/null +++ b/model_results/claude-sonnet-4-5/self_assessment__faq_column_layout.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "self_assessment__faq_column_layout", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 109.74995875358582, + "token_usage": { + "input_tokens": 255584, + "output_tokens": 4426, + "total_tokens": 260010, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 70.72846961021423, + "token_usage": { + "input_tokens": 141898, + "output_tokens": 2968, + "total_tokens": 144866, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 118.35163760185242, + "token_usage": { + "input_tokens": 507057, + "output_tokens": 3783, + "total_tokens": 510840, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 108.76008439064026, + "token_usage": { + "input_tokens": 322068, + "output_tokens": 3631, + "total_tokens": 325699, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/self_assessment__hyperfocus_analysis_report.json b/model_results/claude-sonnet-4-5/self_assessment__hyperfocus_analysis_report.json new file mode 100644 index 00000000..63db49c5 --- /dev/null +++ b/model_results/claude-sonnet-4-5/self_assessment__hyperfocus_analysis_report.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "self_assessment__hyperfocus_analysis_report", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 130.06600856781006, + "token_usage": { + "input_tokens": 210179, + "output_tokens": 5463, + "total_tokens": 215642, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 146.61025142669678, + "token_usage": { + "input_tokens": 624249, + "output_tokens": 7174, + "total_tokens": 631423, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 146.38673877716064, + "token_usage": { + "input_tokens": 605507, + "output_tokens": 6515, + "total_tokens": 612022, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 149.1700623035431, + "token_usage": { + "input_tokens": 578711, + "output_tokens": 6556, + "total_tokens": 585267, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/self_assessment__numbered_list_emojis.json b/model_results/claude-sonnet-4-5/self_assessment__numbered_list_emojis.json new file mode 100644 index 00000000..0809d5ca --- /dev/null +++ b/model_results/claude-sonnet-4-5/self_assessment__numbered_list_emojis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "self_assessment__numbered_list_emojis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 104.32679319381714, + "token_usage": { + "input_tokens": 296811, + "output_tokens": 3844, + "total_tokens": 300655, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 103.34714984893799, + "token_usage": { + "input_tokens": 575570, + "output_tokens": 4214, + "total_tokens": 579784, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 144.49188137054443, + "token_usage": { + "input_tokens": 1089486, + "output_tokens": 4932, + "total_tokens": 1094418, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 111.16063499450684, + "token_usage": { + "input_tokens": 359493, + "output_tokens": 4728, + "total_tokens": 364221, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping__advanced_product_analysis.json b/model_results/claude-sonnet-4-5/shopping__advanced_product_analysis.json new file mode 100644 index 00000000..831d98f5 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping__advanced_product_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping__advanced_product_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 161.12943959236145, + "token_usage": { + "input_tokens": 1485013, + "output_tokens": 3170, + "total_tokens": 1488183, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 157.22969889640808, + "token_usage": { + "input_tokens": 1438632, + "output_tokens": 2688, + "total_tokens": 1441320, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 209.5884816646576, + "token_usage": { + "input_tokens": 1411639, + "output_tokens": 2970, + "total_tokens": 1414609, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 300.6007089614868, + "token_usage": { + "input_tokens": 1441580, + "output_tokens": 2757, + "total_tokens": 1444337, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping__gaming_accessories_analysis.json b/model_results/claude-sonnet-4-5/shopping__gaming_accessories_analysis.json new file mode 100644 index 00000000..5cd52c67 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping__gaming_accessories_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping__gaming_accessories_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 217.76334762573242, + "token_usage": { + "input_tokens": 2311853, + "output_tokens": 4268, + "total_tokens": 2316121, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 323.9741506576538, + "token_usage": { + "input_tokens": 2851047, + "output_tokens": 4572, + "total_tokens": 2855619, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 357.5807902812958, + "token_usage": { + "input_tokens": 2466419, + "output_tokens": 4105, + "total_tokens": 2470524, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 232.4150779247284, + "token_usage": { + "input_tokens": 2342845, + "output_tokens": 4211, + "total_tokens": 2347056, + "reasoning_tokens": 0 + }, + "turn_count": 28 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping__health_routine_optimization.json b/model_results/claude-sonnet-4-5/shopping__health_routine_optimization.json new file mode 100644 index 00000000..b76492ee --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping__health_routine_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping__health_routine_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 198.16809844970703, + "token_usage": { + "input_tokens": 1864471, + "output_tokens": 3945, + "total_tokens": 1868416, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 260.1745903491974, + "token_usage": { + "input_tokens": 2348968, + "output_tokens": 4281, + "total_tokens": 2353249, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 257.822390794754, + "token_usage": { + "input_tokens": 1649060, + "output_tokens": 3450, + "total_tokens": 1652510, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 202.98289275169373, + "token_usage": { + "input_tokens": 1962549, + "output_tokens": 4216, + "total_tokens": 1966765, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping__holiday_baking_competition.json b/model_results/claude-sonnet-4-5/shopping__holiday_baking_competition.json new file mode 100644 index 00000000..4065df42 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping__holiday_baking_competition.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping__holiday_baking_competition", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200895 tokens \\u003e 200000 maximum (tid: 2025093017434377776744150735908)\"},\"type\":\"error\"}", + "execution_time": 208.97467589378357, + "token_usage": { + "input_tokens": 2569627, + "output_tokens": 3522, + "total_tokens": 2573149, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 206303 tokens \\u003e 200000 maximum (tid: 2025100103595095925235059259628)\"},\"type\":\"error\"}", + "execution_time": 278.1035141944885, + "token_usage": { + "input_tokens": 2491902, + "output_tokens": 4163, + "total_tokens": 2496065, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 202595 tokens \\u003e 200000 maximum (tid: 2025100110274095473545329985703)\"},\"type\":\"error\"}", + "execution_time": 365.9616105556488, + "token_usage": { + "input_tokens": 2586401, + "output_tokens": 3482, + "total_tokens": 2589883, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 203433 tokens \\u003e 200000 maximum (tid: 2025100111461111930268441732950)\"},\"type\":\"error\"}", + "execution_time": 298.0338854789734, + "token_usage": { + "input_tokens": 2449369, + "output_tokens": 4732, + "total_tokens": 2454101, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping__multi_category_budget_analysis.json b/model_results/claude-sonnet-4-5/shopping__multi_category_budget_analysis.json new file mode 100644 index 00000000..f273dec0 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping__multi_category_budget_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping__multi_category_budget_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 206140 tokens \\u003e 200000 maximum (tid: 2025093017481419090799882571876)\"},\"type\":\"error\"}", + "execution_time": 211.2567048072815, + "token_usage": { + "input_tokens": 2511826, + "output_tokens": 3745, + "total_tokens": 2515571, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200402 tokens \\u003e 200000 maximum (tid: 2025100104052497827292071938060)\"},\"type\":\"error\"}", + "execution_time": 267.4960823059082, + "token_usage": { + "input_tokens": 2592236, + "output_tokens": 3928, + "total_tokens": 2596164, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 188.2953622341156, + "token_usage": { + "input_tokens": 2229840, + "output_tokens": 3706, + "total_tokens": 2233546, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 202023 tokens \\u003e 200000 maximum (tid: 2025100111513648123759966073979)\"},\"type\":\"error\"}", + "execution_time": 268.89570021629333, + "token_usage": { + "input_tokens": 2467668, + "output_tokens": 3609, + "total_tokens": 2471277, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping__printer_keyboard_search.json b/model_results/claude-sonnet-4-5/shopping__printer_keyboard_search.json new file mode 100644 index 00000000..d7f03fd3 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping__printer_keyboard_search.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping__printer_keyboard_search", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 133.2373776435852, + "token_usage": { + "input_tokens": 839378, + "output_tokens": 2754, + "total_tokens": 842132, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 171.02756547927856, + "token_usage": { + "input_tokens": 1193888, + "output_tokens": 3251, + "total_tokens": 1197139, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 161.84360098838806, + "token_usage": { + "input_tokens": 1478641, + "output_tokens": 3198, + "total_tokens": 1481839, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 154.02386164665222, + "token_usage": { + "input_tokens": 1312878, + "output_tokens": 2796, + "total_tokens": 1315674, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping__running_shoes_purchase.json b/model_results/claude-sonnet-4-5/shopping__running_shoes_purchase.json new file mode 100644 index 00000000..6bc5873d --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping__running_shoes_purchase.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping__running_shoes_purchase", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 207826 tokens \\u003e 200000 maximum (tid: 2025093017544827073042577958600)\"},\"type\":\"error\"}", + "execution_time": 138.48571109771729, + "token_usage": { + "input_tokens": 1231565, + "output_tokens": 2616, + "total_tokens": 1234181, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 168.50810718536377, + "token_usage": { + "input_tokens": 1007709, + "output_tokens": 3169, + "total_tokens": 1010878, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 110.85117816925049, + "token_usage": { + "input_tokens": 793148, + "output_tokens": 2363, + "total_tokens": 795511, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 205531 tokens \\u003e 200000 maximum (tid: 2025100111583290560553648306134)\"},\"type\":\"error\"}", + "execution_time": 138.63545441627502, + "token_usage": { + "input_tokens": 1399851, + "output_tokens": 2543, + "total_tokens": 1402394, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping_admin__customer_segmentation_setup.json b/model_results/claude-sonnet-4-5/shopping_admin__customer_segmentation_setup.json new file mode 100644 index 00000000..e7e3a33c --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping_admin__customer_segmentation_setup.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping_admin__customer_segmentation_setup", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 124.42920970916748, + "token_usage": { + "input_tokens": 662531, + "output_tokens": 2375, + "total_tokens": 664906, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 156.69361281394958, + "token_usage": { + "input_tokens": 650714, + "output_tokens": 2664, + "total_tokens": 653378, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 129.42514824867249, + "token_usage": { + "input_tokens": 461367, + "output_tokens": 2517, + "total_tokens": 463884, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 138.74757146835327, + "token_usage": { + "input_tokens": 663494, + "output_tokens": 2452, + "total_tokens": 665946, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping_admin__fitness_promotion_strategy.json b/model_results/claude-sonnet-4-5/shopping_admin__fitness_promotion_strategy.json new file mode 100644 index 00000000..955293d4 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping_admin__fitness_promotion_strategy.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping_admin__fitness_promotion_strategy", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 215970 tokens \\u003e 200000 maximum (tid: 2025093017595369093441982686905)\"},\"type\":\"error\"}", + "execution_time": 94.99269604682922, + "token_usage": { + "input_tokens": 582810, + "output_tokens": 1486, + "total_tokens": 584296, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 216475 tokens \\u003e 200000 maximum (tid: 2025100104190470563583605455844)\"},\"type\":\"error\"}", + "execution_time": 112.86685919761658, + "token_usage": { + "input_tokens": 581733, + "output_tokens": 1616, + "total_tokens": 583349, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200959 tokens \\u003e 200000 maximum (tid: 2025100110430663866266498883247)\"},\"type\":\"error\"}", + "execution_time": 101.56730937957764, + "token_usage": { + "input_tokens": 527583, + "output_tokens": 1325, + "total_tokens": 528908, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 216059 tokens \\u003e 200000 maximum (tid: 2025100112034419405400454256316)\"},\"type\":\"error\"}", + "execution_time": 91.0614743232727, + "token_usage": { + "input_tokens": 575370, + "output_tokens": 1521, + "total_tokens": 576891, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping_admin__marketing_customer_analysis.json b/model_results/claude-sonnet-4-5/shopping_admin__marketing_customer_analysis.json new file mode 100644 index 00000000..30af2fd6 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping_admin__marketing_customer_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping_admin__marketing_customer_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 193.00646543502808, + "token_usage": { + "input_tokens": 1217047, + "output_tokens": 4250, + "total_tokens": 1221297, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 220.85941910743713, + "token_usage": { + "input_tokens": 1530795, + "output_tokens": 4139, + "total_tokens": 1534934, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 191.45992827415466, + "token_usage": { + "input_tokens": 1494771, + "output_tokens": 4180, + "total_tokens": 1498951, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 257.8118076324463, + "token_usage": { + "input_tokens": 1537487, + "output_tokens": 4049, + "total_tokens": 1541536, + "reasoning_tokens": 0 + }, + "turn_count": 27 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping_admin__ny_expansion_analysis.json b/model_results/claude-sonnet-4-5/shopping_admin__ny_expansion_analysis.json new file mode 100644 index 00000000..4c177381 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping_admin__ny_expansion_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping_admin__ny_expansion_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 205243 tokens \\u003e 200000 maximum (tid: 202509301805368543753586703595)\"},\"type\":\"error\"}", + "execution_time": 66.26638674736023, + "token_usage": { + "input_tokens": 407541, + "output_tokens": 1122, + "total_tokens": 408663, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 205325 tokens \\u003e 200000 maximum (tid: 2025100104252595547047487591667)\"},\"type\":\"error\"}", + "execution_time": 70.09230470657349, + "token_usage": { + "input_tokens": 278697, + "output_tokens": 1141, + "total_tokens": 279838, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 202146 tokens \\u003e 200000 maximum (tid: 2025100110504614609050630807728)\"},\"type\":\"error\"}", + "execution_time": 187.66044521331787, + "token_usage": { + "input_tokens": 2324975, + "output_tokens": 3051, + "total_tokens": 2328026, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 221.8924160003662, + "token_usage": { + "input_tokens": 2512099, + "output_tokens": 3918, + "total_tokens": 2516017, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping_admin__products_sales_analysis.json b/model_results/claude-sonnet-4-5/shopping_admin__products_sales_analysis.json new file mode 100644 index 00000000..b6c39177 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping_admin__products_sales_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping_admin__products_sales_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 201079 tokens \\u003e 200000 maximum (tid: 2025093018072429024005353080696)\"},\"type\":\"error\"}", + "execution_time": 78.81223106384277, + "token_usage": { + "input_tokens": 371201, + "output_tokens": 1252, + "total_tokens": 372453, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 201012 tokens \\u003e 200000 maximum (tid: 2025100104272653832071144001689)\"},\"type\":\"error\"}", + "execution_time": 79.45995903015137, + "token_usage": { + "input_tokens": 367479, + "output_tokens": 1217, + "total_tokens": 368696, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200519 tokens \\u003e 200000 maximum (tid: 2025100110525193258628161760257)\"},\"type\":\"error\"}", + "execution_time": 94.72774529457092, + "token_usage": { + "input_tokens": 370438, + "output_tokens": 1256, + "total_tokens": 371694, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200869 tokens \\u003e 200000 maximum (tid: 2025100112144712402964779008781)\"},\"type\":\"error\"}", + "execution_time": 73.70070576667786, + "token_usage": { + "input_tokens": 369694, + "output_tokens": 1093, + "total_tokens": 370787, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping_admin__sales_inventory_analysis.json b/model_results/claude-sonnet-4-5/shopping_admin__sales_inventory_analysis.json new file mode 100644 index 00000000..69826212 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping_admin__sales_inventory_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping_admin__sales_inventory_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200605 tokens \\u003e 200000 maximum (tid: 2025093018091875798349441920192)\"},\"type\":\"error\"}", + "execution_time": 79.55335783958435, + "token_usage": { + "input_tokens": 541427, + "output_tokens": 1256, + "total_tokens": 542683, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200284 tokens \\u003e 200000 maximum (tid: 2025100104292247914175571911268)\"},\"type\":\"error\"}", + "execution_time": 85.35966205596924, + "token_usage": { + "input_tokens": 538155, + "output_tokens": 1222, + "total_tokens": 539377, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200282 tokens \\u003e 200000 maximum (tid: 2025100110543997349135824770456)\"},\"type\":\"error\"}", + "execution_time": 77.52313089370728, + "token_usage": { + "input_tokens": 540754, + "output_tokens": 1172, + "total_tokens": 541926, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 200675 tokens \\u003e 200000 maximum (tid: 2025100112163884091188481329782)\"},\"type\":\"error\"}", + "execution_time": 87.099769115448, + "token_usage": { + "input_tokens": 544108, + "output_tokens": 1320, + "total_tokens": 545428, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/shopping_admin__search_filtering_operations.json b/model_results/claude-sonnet-4-5/shopping_admin__search_filtering_operations.json new file mode 100644 index 00000000..dbf211f5 --- /dev/null +++ b/model_results/claude-sonnet-4-5/shopping_admin__search_filtering_operations.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "shopping_admin__search_filtering_operations", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 240.16096234321594, + "token_usage": { + "input_tokens": 1390423, + "output_tokens": 5198, + "total_tokens": 1395621, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 218.5385138988495, + "token_usage": { + "input_tokens": 1081965, + "output_tokens": 5329, + "total_tokens": 1087294, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 254.2272596359253, + "token_usage": { + "input_tokens": 1481207, + "output_tokens": 5645, + "total_tokens": 1486852, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 262.48115944862366, + "token_usage": { + "input_tokens": 1563942, + "output_tokens": 4976, + "total_tokens": 1568918, + "reasoning_tokens": 0 + }, + "turn_count": 36 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/sports__baseball_player_analysis.json b/model_results/claude-sonnet-4-5/sports__baseball_player_analysis.json new file mode 100644 index 00000000..ad62d296 --- /dev/null +++ b/model_results/claude-sonnet-4-5/sports__baseball_player_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "sports__baseball_player_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 111.87357664108276, + "token_usage": { + "input_tokens": 441708, + "output_tokens": 5164, + "total_tokens": 446872, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 147.65282464027405, + "token_usage": { + "input_tokens": 690167, + "output_tokens": 6269, + "total_tokens": 696436, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 122.88530802726746, + "token_usage": { + "input_tokens": 435252, + "output_tokens": 5296, + "total_tokens": 440548, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 132.57741951942444, + "token_usage": { + "input_tokens": 551234, + "output_tokens": 6096, + "total_tokens": 557330, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/sports__participant_report_optimization.json b/model_results/claude-sonnet-4-5/sports__participant_report_optimization.json new file mode 100644 index 00000000..9d184a2d --- /dev/null +++ b/model_results/claude-sonnet-4-5/sports__participant_report_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "sports__participant_report_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 84.42110967636108, + "token_usage": { + "input_tokens": 188109, + "output_tokens": 3587, + "total_tokens": 191696, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 84.15524506568909, + "token_usage": { + "input_tokens": 211772, + "output_tokens": 3685, + "total_tokens": 215457, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 114.46356654167175, + "token_usage": { + "input_tokens": 288767, + "output_tokens": 5037, + "total_tokens": 293804, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 84.08264136314392, + "token_usage": { + "input_tokens": 177526, + "output_tokens": 3980, + "total_tokens": 181506, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/sports__team_roster_management.json b/model_results/claude-sonnet-4-5/sports__team_roster_management.json new file mode 100644 index 00000000..bcd3b06e --- /dev/null +++ b/model_results/claude-sonnet-4-5/sports__team_roster_management.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "sports__team_roster_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 124.91776180267334, + "token_usage": { + "input_tokens": 450145, + "output_tokens": 4996, + "total_tokens": 455141, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 111.1750020980835, + "token_usage": { + "input_tokens": 393854, + "output_tokens": 4447, + "total_tokens": 398301, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 135.88168215751648, + "token_usage": { + "input_tokens": 432908, + "output_tokens": 5114, + "total_tokens": 438022, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 119.40657472610474, + "token_usage": { + "input_tokens": 415448, + "output_tokens": 4817, + "total_tokens": 420265, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/standard_operating_procedure__deployment_process_sop.json b/model_results/claude-sonnet-4-5/standard_operating_procedure__deployment_process_sop.json new file mode 100644 index 00000000..11d4f1dc --- /dev/null +++ b/model_results/claude-sonnet-4-5/standard_operating_procedure__deployment_process_sop.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "standard_operating_procedure__deployment_process_sop", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 317.0396327972412, + "token_usage": { + "input_tokens": 2450141, + "output_tokens": 8400, + "total_tokens": 2458541, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 198.16494750976562, + "token_usage": { + "input_tokens": 646837, + "output_tokens": 5827, + "total_tokens": 652664, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 358.2421863079071, + "token_usage": { + "input_tokens": 1396044, + "output_tokens": 11315, + "total_tokens": 1407359, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 292.14235186576843, + "token_usage": { + "input_tokens": 1633368, + "output_tokens": 7705, + "total_tokens": 1641073, + "reasoning_tokens": 0 + }, + "turn_count": 27 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/standard_operating_procedure__section_organization.json b/model_results/claude-sonnet-4-5/standard_operating_procedure__section_organization.json new file mode 100644 index 00000000..531d5c02 --- /dev/null +++ b/model_results/claude-sonnet-4-5/standard_operating_procedure__section_organization.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "standard_operating_procedure__section_organization", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 129.71568870544434, + "token_usage": { + "input_tokens": 283715, + "output_tokens": 4008, + "total_tokens": 287723, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 237.5986340045929, + "token_usage": { + "input_tokens": 625378, + "output_tokens": 7882, + "total_tokens": 633260, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 126.77526426315308, + "token_usage": { + "input_tokens": 210289, + "output_tokens": 4457, + "total_tokens": 214746, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 157.80155682563782, + "token_usage": { + "input_tokens": 596844, + "output_tokens": 3893, + "total_tokens": 600737, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/student_database__duplicate_name.json b/model_results/claude-sonnet-4-5/student_database__duplicate_name.json new file mode 100644 index 00000000..b13a084f --- /dev/null +++ b/model_results/claude-sonnet-4-5/student_database__duplicate_name.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "student_database__duplicate_name", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 30.72661805152893, + "token_usage": { + "input_tokens": 55254, + "output_tokens": 1379, + "total_tokens": 56633, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 20.064249753952026, + "token_usage": { + "input_tokens": 15288, + "output_tokens": 823, + "total_tokens": 16111, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 37.03082346916199, + "token_usage": { + "input_tokens": 79264, + "output_tokens": 1526, + "total_tokens": 80790, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 35.16418647766113, + "token_usage": { + "input_tokens": 25075, + "output_tokens": 1205, + "total_tokens": 26280, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/student_database__english_talent.json b/model_results/claude-sonnet-4-5/student_database__english_talent.json new file mode 100644 index 00000000..3ff68965 --- /dev/null +++ b/model_results/claude-sonnet-4-5/student_database__english_talent.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "student_database__english_talent", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 275.5563473701477, + "token_usage": { + "input_tokens": 1328835, + "output_tokens": 18869, + "total_tokens": 1347704, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 218.489684343338, + "token_usage": { + "input_tokens": 943517, + "output_tokens": 17222, + "total_tokens": 960739, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 225.853257894516, + "token_usage": { + "input_tokens": 808857, + "output_tokens": 17850, + "total_tokens": 826707, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 317.272718667984, + "token_usage": { + "input_tokens": 1063621, + "output_tokens": 18079, + "total_tokens": 1081700, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/student_database__gradebased_score.json b/model_results/claude-sonnet-4-5/student_database__gradebased_score.json new file mode 100644 index 00000000..b6c0a9ae --- /dev/null +++ b/model_results/claude-sonnet-4-5/student_database__gradebased_score.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "student_database__gradebased_score", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 321.98061299324036, + "token_usage": { + "input_tokens": 745075, + "output_tokens": 17943, + "total_tokens": 763018, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 314.3202350139618, + "token_usage": { + "input_tokens": 994063, + "output_tokens": 20029, + "total_tokens": 1014092, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 345.93722653388977, + "token_usage": { + "input_tokens": 890404, + "output_tokens": 26356, + "total_tokens": 916760, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 214.66334557533264, + "token_usage": { + "input_tokens": 627392, + "output_tokens": 13631, + "total_tokens": 641023, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/team_projects__priority_tasks_table.json b/model_results/claude-sonnet-4-5/team_projects__priority_tasks_table.json new file mode 100644 index 00000000..90e23f1b --- /dev/null +++ b/model_results/claude-sonnet-4-5/team_projects__priority_tasks_table.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "team_projects__priority_tasks_table", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 47.102660179138184, + "token_usage": { + "input_tokens": 99242, + "output_tokens": 2090, + "total_tokens": 101332, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 43.743205070495605, + "token_usage": { + "input_tokens": 78028, + "output_tokens": 1922, + "total_tokens": 79950, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 132.97260212898254, + "token_usage": { + "input_tokens": 499871, + "output_tokens": 5340, + "total_tokens": 505211, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 138.0257544517517, + "token_usage": { + "input_tokens": 515224, + "output_tokens": 3768, + "total_tokens": 518992, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/team_projects__swap_tasks.json b/model_results/claude-sonnet-4-5/team_projects__swap_tasks.json new file mode 100644 index 00000000..dbe1ad0b --- /dev/null +++ b/model_results/claude-sonnet-4-5/team_projects__swap_tasks.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "team_projects__swap_tasks", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 106.19912338256836, + "token_usage": { + "input_tokens": 219582, + "output_tokens": 4346, + "total_tokens": 223928, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 88.05694580078125, + "token_usage": { + "input_tokens": 205850, + "output_tokens": 3919, + "total_tokens": 209769, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 699.3016955852509, + "token_usage": { + "input_tokens": 287309, + "output_tokens": 3144, + "total_tokens": 290453, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 109.31528568267822, + "token_usage": { + "input_tokens": 180459, + "output_tokens": 3606, + "total_tokens": 184065, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/threestudio__code_locating.json b/model_results/claude-sonnet-4-5/threestudio__code_locating.json new file mode 100644 index 00000000..bd114880 --- /dev/null +++ b/model_results/claude-sonnet-4-5/threestudio__code_locating.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "threestudio__code_locating", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 46.14034676551819, + "token_usage": { + "input_tokens": 153442, + "output_tokens": 1575, + "total_tokens": 155017, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 31.756644248962402, + "token_usage": { + "input_tokens": 101496, + "output_tokens": 982, + "total_tokens": 102478, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 24.856565237045288, + "token_usage": { + "input_tokens": 72656, + "output_tokens": 792, + "total_tokens": 73448, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 31.87791347503662, + "token_usage": { + "input_tokens": 89543, + "output_tokens": 1010, + "total_tokens": 90553, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/threestudio__output_analysis.json b/model_results/claude-sonnet-4-5/threestudio__output_analysis.json new file mode 100644 index 00000000..b6b18a37 --- /dev/null +++ b/model_results/claude-sonnet-4-5/threestudio__output_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "threestudio__output_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 40.25979995727539, + "token_usage": { + "input_tokens": 94885, + "output_tokens": 1410, + "total_tokens": 96295, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 37.768054723739624, + "token_usage": { + "input_tokens": 94832, + "output_tokens": 1314, + "total_tokens": 96146, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 34.88453793525696, + "token_usage": { + "input_tokens": 94442, + "output_tokens": 1011, + "total_tokens": 95453, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 35.897154569625854, + "token_usage": { + "input_tokens": 94367, + "output_tokens": 1173, + "total_tokens": 95540, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/threestudio__requirements_completion.json b/model_results/claude-sonnet-4-5/threestudio__requirements_completion.json new file mode 100644 index 00000000..f3695445 --- /dev/null +++ b/model_results/claude-sonnet-4-5/threestudio__requirements_completion.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "threestudio__requirements_completion", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 96.80678296089172, + "token_usage": { + "input_tokens": 490331, + "output_tokens": 3051, + "total_tokens": 493382, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 120.20909237861633, + "token_usage": { + "input_tokens": 819828, + "output_tokens": 3974, + "total_tokens": 823802, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 59.96433162689209, + "token_usage": { + "input_tokens": 412603, + "output_tokens": 1941, + "total_tokens": 414544, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 138.27081394195557, + "token_usage": { + "input_tokens": 1028510, + "output_tokens": 4324, + "total_tokens": 1032834, + "reasoning_tokens": 0 + }, + "turn_count": 31 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/toronto_guide__change_color.json b/model_results/claude-sonnet-4-5/toronto_guide__change_color.json new file mode 100644 index 00000000..deec6a89 --- /dev/null +++ b/model_results/claude-sonnet-4-5/toronto_guide__change_color.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "toronto_guide__change_color", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 257.97811627388, + "token_usage": { + "input_tokens": 704246, + "output_tokens": 10728, + "total_tokens": 714974, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 249.97449684143066, + "token_usage": { + "input_tokens": 596650, + "output_tokens": 9249, + "total_tokens": 605899, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 177.7481129169464, + "token_usage": { + "input_tokens": 635551, + "output_tokens": 5751, + "total_tokens": 641302, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 219.6611545085907, + "token_usage": { + "input_tokens": 442376, + "output_tokens": 7734, + "total_tokens": 450110, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/toronto_guide__weekend_adventure_planner.json b/model_results/claude-sonnet-4-5/toronto_guide__weekend_adventure_planner.json new file mode 100644 index 00000000..e9601838 --- /dev/null +++ b/model_results/claude-sonnet-4-5/toronto_guide__weekend_adventure_planner.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "notion", + "task": "toronto_guide__weekend_adventure_planner", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 70.25224924087524, + "token_usage": { + "input_tokens": 122081, + "output_tokens": 3001, + "total_tokens": 125082, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 84.13027787208557, + "token_usage": { + "input_tokens": 280673, + "output_tokens": 3512, + "total_tokens": 284185, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 138.0076162815094, + "token_usage": { + "input_tokens": 921051, + "output_tokens": 4131, + "total_tokens": 925182, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 123.71289491653442, + "token_usage": { + "input_tokens": 556893, + "output_tokens": 4147, + "total_tokens": 561040, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/vectors__dba_vector_analysis.json b/model_results/claude-sonnet-4-5/vectors__dba_vector_analysis.json new file mode 100644 index 00000000..67cffd40 --- /dev/null +++ b/model_results/claude-sonnet-4-5/vectors__dba_vector_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "postgres", + "task": "vectors__dba_vector_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 341.4015967845917, + "token_usage": { + "input_tokens": 711626, + "output_tokens": 17687, + "total_tokens": 729313, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 252.53261613845825, + "token_usage": { + "input_tokens": 499625, + "output_tokens": 14030, + "total_tokens": 513655, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 299.88823342323303, + "token_usage": { + "input_tokens": 1123777, + "output_tokens": 13947, + "total_tokens": 1137724, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 294.0345697402954, + "token_usage": { + "input_tokens": 929872, + "output_tokens": 14959, + "total_tokens": 944831, + "reasoning_tokens": 0 + }, + "turn_count": 35 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/votenet__dataset_comparison.json b/model_results/claude-sonnet-4-5/votenet__dataset_comparison.json new file mode 100644 index 00000000..5adfd4d6 --- /dev/null +++ b/model_results/claude-sonnet-4-5/votenet__dataset_comparison.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "votenet__dataset_comparison", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 47.45456051826477, + "token_usage": { + "input_tokens": 181071, + "output_tokens": 1707, + "total_tokens": 182778, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 104.14181900024414, + "token_usage": { + "input_tokens": 371285, + "output_tokens": 5042, + "total_tokens": 376327, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 167.68658542633057, + "token_usage": { + "input_tokens": 659712, + "output_tokens": 8726, + "total_tokens": 668438, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 47.27187442779541, + "token_usage": { + "input_tokens": 220280, + "output_tokens": 1734, + "total_tokens": 222014, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/votenet__debugging.json b/model_results/claude-sonnet-4-5/votenet__debugging.json new file mode 100644 index 00000000..3f7964f8 --- /dev/null +++ b/model_results/claude-sonnet-4-5/votenet__debugging.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "votenet__debugging", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 54.12547707557678, + "token_usage": { + "input_tokens": 143767, + "output_tokens": 2137, + "total_tokens": 145904, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 55.069950342178345, + "token_usage": { + "input_tokens": 98578, + "output_tokens": 2417, + "total_tokens": 100995, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 65.36676406860352, + "token_usage": { + "input_tokens": 99070, + "output_tokens": 2559, + "total_tokens": 101629, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 44.07355260848999, + "token_usage": { + "input_tokens": 63931, + "output_tokens": 1795, + "total_tokens": 65726, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/votenet__requirements_writing.json b/model_results/claude-sonnet-4-5/votenet__requirements_writing.json new file mode 100644 index 00000000..a4eb36aa --- /dev/null +++ b/model_results/claude-sonnet-4-5/votenet__requirements_writing.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "filesystem", + "task": "votenet__requirements_writing", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 59.96407222747803, + "token_usage": { + "input_tokens": 402715, + "output_tokens": 2229, + "total_tokens": 404944, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 58.19667339324951, + "token_usage": { + "input_tokens": 356290, + "output_tokens": 2118, + "total_tokens": 358408, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 52.0265839099884, + "token_usage": { + "input_tokens": 294978, + "output_tokens": 1965, + "total_tokens": 296943, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 52.09568786621094, + "token_usage": { + "input_tokens": 231175, + "output_tokens": 1812, + "total_tokens": 232987, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/web_search__birth_of_arvinxu.json b/model_results/claude-sonnet-4-5/web_search__birth_of_arvinxu.json new file mode 100644 index 00000000..7d52a4d5 --- /dev/null +++ b/model_results/claude-sonnet-4-5/web_search__birth_of_arvinxu.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "web_search__birth_of_arvinxu", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 103.19063949584961, + "token_usage": { + "input_tokens": 434800, + "output_tokens": 1896, + "total_tokens": 436696, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 96.57188034057617, + "token_usage": { + "input_tokens": 393469, + "output_tokens": 1677, + "total_tokens": 395146, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 143.63067388534546, + "token_usage": { + "input_tokens": 780355, + "output_tokens": 2584, + "total_tokens": 782939, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 119.75165700912476, + "token_usage": { + "input_tokens": 669873, + "output_tokens": 1907, + "total_tokens": 671780, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/claude-sonnet-4-5/web_search__r1_arxiv.json b/model_results/claude-sonnet-4-5/web_search__r1_arxiv.json new file mode 100644 index 00000000..0aa4e8f6 --- /dev/null +++ b/model_results/claude-sonnet-4-5/web_search__r1_arxiv.json @@ -0,0 +1,55 @@ +{ + "model": "claude-sonnet-4-5", + "service": "playwright", + "task": "web_search__r1_arxiv", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 76.32272362709045, + "token_usage": { + "input_tokens": 477361, + "output_tokens": 1529, + "total_tokens": 478890, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 201770 tokens \\u003e 200000 maximum (tid: 2025093012262817082288133410919)\"},\"type\":\"error\"}", + "execution_time": 99.96746134757996, + "token_usage": { + "input_tokens": 817476, + "output_tokens": 1428, + "total_tokens": 818904, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 205852 tokens \\u003e 200000 maximum (tid: 2025093014071123142688728401326)\"},\"type\":\"error\"}", + "execution_time": 93.44523286819458, + "token_usage": { + "input_tokens": 931601, + "output_tokens": 1443, + "total_tokens": 933044, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: AnthropicError - {\"error\":{\"type\":\"invalid_request_error\",\"message\":\"prompt is too long: 203693 tokens \\u003e 200000 maximum (tid: 2025093015574414226066927005114)\"},\"type\":\"error\"}", + "execution_time": 81.71313190460205, + "token_usage": { + "input_tokens": 774558, + "output_tokens": 1249, + "total_tokens": 775807, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/build_your_own_x__find_commit_date.json b/model_results/deepseek-v3-1-terminus-thinking/build_your_own_x__find_commit_date.json new file mode 100644 index 00000000..b7b73b5c --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/build_your_own_x__find_commit_date.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "build_your_own_x__find_commit_date", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 357.807208776474, + "token_usage": { + "input_tokens": 1602096, + "output_tokens": 2324, + "total_tokens": 1604420, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 775.9163918495178, + "token_usage": { + "input_tokens": 295890, + "output_tokens": 19418, + "total_tokens": 315308, + "reasoning_tokens": 18494 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 680.9810349941254, + "token_usage": { + "input_tokens": 232305, + "output_tokens": 14071, + "total_tokens": 246376, + "reasoning_tokens": 13575 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 940.7171416282654, + "token_usage": { + "input_tokens": 365629, + "output_tokens": 24970, + "total_tokens": 390599, + "reasoning_tokens": 24116 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/build_your_own_x__find_rag_commit.json b/model_results/deepseek-v3-1-terminus-thinking/build_your_own_x__find_rag_commit.json new file mode 100644 index 00000000..f6c1cfbf --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/build_your_own_x__find_rag_commit.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "build_your_own_x__find_rag_commit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 260.5582346916199, + "token_usage": { + "input_tokens": 819475, + "output_tokens": 1957, + "total_tokens": 821432, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 1811.415769815445, + "token_usage": { + "input_tokens": 823100, + "output_tokens": 44703, + "total_tokens": 867803, + "reasoning_tokens": 43139 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1345.8292202949524, + "token_usage": { + "input_tokens": 966167, + "output_tokens": 22768, + "total_tokens": 988935, + "reasoning_tokens": 21494 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1041.5495312213898, + "token_usage": { + "input_tokens": 562922, + "output_tokens": 27196, + "total_tokens": 590118, + "reasoning_tokens": 26315 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/chinook__customer_data_migration.json b/model_results/deepseek-v3-1-terminus-thinking/chinook__customer_data_migration.json new file mode 100644 index 00000000..0d4ea78a --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/chinook__customer_data_migration.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "chinook__customer_data_migration", + "runs": { + "run-1": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 9 (char 8)", + "execution_time": 226.29387187957764, + "token_usage": { + "input_tokens": 61388, + "output_tokens": 4272, + "total_tokens": 65660, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 9 (char 8)", + "execution_time": 208.74509859085083, + "token_usage": { + "input_tokens": 61349, + "output_tokens": 4254, + "total_tokens": 65603, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 9 (char 8)", + "execution_time": 204.9047417640686, + "token_usage": { + "input_tokens": 74894, + "output_tokens": 4276, + "total_tokens": 79170, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 11 (char 10)", + "execution_time": 444.83052468299866, + "token_usage": { + "input_tokens": 24824, + "output_tokens": 13065, + "total_tokens": 37889, + "reasoning_tokens": 8912 + }, + "turn_count": 2 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/chinook__employee_hierarchy_management.json b/model_results/deepseek-v3-1-terminus-thinking/chinook__employee_hierarchy_management.json new file mode 100644 index 00000000..5092e9cc --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/chinook__employee_hierarchy_management.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "chinook__employee_hierarchy_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 209.23206996917725, + "token_usage": { + "input_tokens": 245875, + "output_tokens": 2205, + "total_tokens": 248080, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 204.79339170455933, + "token_usage": { + "input_tokens": 246121, + "output_tokens": 2232, + "total_tokens": 248353, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 184.4407684803009, + "token_usage": { + "input_tokens": 204464, + "output_tokens": 2324, + "total_tokens": 206788, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 298.06419038772583, + "token_usage": { + "input_tokens": 13895, + "output_tokens": 8093, + "total_tokens": 21988, + "reasoning_tokens": 5864 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/chinook__sales_and_music_charts.json b/model_results/deepseek-v3-1-terminus-thinking/chinook__sales_and_music_charts.json new file mode 100644 index 00000000..1606e63a --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/chinook__sales_and_music_charts.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "chinook__sales_and_music_charts", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 339.96108889579773, + "token_usage": { + "input_tokens": 428898, + "output_tokens": 4524, + "total_tokens": 433422, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 279.29951310157776, + "token_usage": { + "input_tokens": 337123, + "output_tokens": 3812, + "total_tokens": 340935, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 214.09886193275452, + "token_usage": { + "input_tokens": 216495, + "output_tokens": 2911, + "total_tokens": 219406, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 32.553945541381836, + "token_usage": { + "input_tokens": 2683, + "output_tokens": 780, + "total_tokens": 3463, + "reasoning_tokens": 749 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/claude-code__automated_changelog_generation.json b/model_results/deepseek-v3-1-terminus-thinking/claude-code__automated_changelog_generation.json new file mode 100644 index 00000000..229ab7d1 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/claude-code__automated_changelog_generation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "claude-code__automated_changelog_generation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 328.56732177734375, + "token_usage": { + "input_tokens": 486413, + "output_tokens": 5640, + "total_tokens": 492053, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 335.2862730026245, + "token_usage": { + "input_tokens": 193716, + "output_tokens": 7517, + "total_tokens": 201233, + "reasoning_tokens": 3380 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 601.6498417854309, + "token_usage": { + "input_tokens": 185533, + "output_tokens": 14256, + "total_tokens": 199789, + "reasoning_tokens": 11439 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 69.64990425109863, + "token_usage": { + "input_tokens": 24123, + "output_tokens": 1865, + "total_tokens": 25988, + "reasoning_tokens": 703 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/claude-code__claude_collaboration_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/claude-code__claude_collaboration_analysis.json new file mode 100644 index 00000000..36e9430b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/claude-code__claude_collaboration_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "claude-code__claude_collaboration_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 93.30665850639343, + "token_usage": { + "input_tokens": 260947, + "output_tokens": 1143, + "total_tokens": 262090, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 730.9451944828033, + "token_usage": { + "input_tokens": 224095, + "output_tokens": 19142, + "total_tokens": 243237, + "reasoning_tokens": 18435 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 2150.869268655777, + "token_usage": { + "input_tokens": 371407, + "output_tokens": 53642, + "total_tokens": 425049, + "reasoning_tokens": 52572 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 568.0564692020416, + "token_usage": { + "input_tokens": 224089, + "output_tokens": 16239, + "total_tokens": 240328, + "reasoning_tokens": 15616 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/claude-code__critical_issue_hotfix_workflow.json b/model_results/deepseek-v3-1-terminus-thinking/claude-code__critical_issue_hotfix_workflow.json new file mode 100644 index 00000000..eb125955 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/claude-code__critical_issue_hotfix_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "claude-code__critical_issue_hotfix_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 177.06695079803467, + "token_usage": { + "input_tokens": 220752, + "output_tokens": 2584, + "total_tokens": 223336, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 634.8860394954681, + "token_usage": { + "input_tokens": 141976, + "output_tokens": 16958, + "total_tokens": 158934, + "reasoning_tokens": 15301 + }, + "turn_count": 5 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 701.3802073001862, + "token_usage": { + "input_tokens": 168780, + "output_tokens": 17235, + "total_tokens": 186015, + "reasoning_tokens": 15356 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 362.0613806247711, + "token_usage": { + "input_tokens": 79970, + "output_tokens": 10124, + "total_tokens": 90094, + "reasoning_tokens": 8270 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/claude-code__feature_commit_tracking.json b/model_results/deepseek-v3-1-terminus-thinking/claude-code__feature_commit_tracking.json new file mode 100644 index 00000000..cc18f0ff --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/claude-code__feature_commit_tracking.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "claude-code__feature_commit_tracking", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 287.90511178970337, + "token_usage": { + "input_tokens": 1373753, + "output_tokens": 2258, + "total_tokens": 1376011, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092700031060113967776450776)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 871.6150290966034, + "token_usage": { + "input_tokens": 189344, + "output_tokens": 18399, + "total_tokens": 207743, + "reasoning_tokens": 17934 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1428.207557439804, + "token_usage": { + "input_tokens": 805277, + "output_tokens": 27661, + "total_tokens": 832938, + "reasoning_tokens": 26100 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 20250927202556165590358993002)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 522.0259490013123, + "token_usage": { + "input_tokens": 275952, + "output_tokens": 12773, + "total_tokens": 288725, + "reasoning_tokens": 12405 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/claude-code__label_color_standardization.json b/model_results/deepseek-v3-1-terminus-thinking/claude-code__label_color_standardization.json new file mode 100644 index 00000000..4bc23025 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/claude-code__label_color_standardization.json @@ -0,0 +1,50 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "claude-code__label_color_standardization", + "runs": { + "run-1": { + "success": false, + "error_message": "Execution timed out after 3600 seconds", + "execution_time": 30.060205221176147, + "token_usage": {}, + "turn_count": 0 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 272.34059476852417, + "token_usage": { + "input_tokens": 24064, + "output_tokens": 7793, + "total_tokens": 31857, + "reasoning_tokens": 7728 + }, + "turn_count": 1 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1245.5912964344025, + "token_usage": { + "input_tokens": 383667, + "output_tokens": 26100, + "total_tokens": 409767, + "reasoning_tokens": 24258 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092720303592045450058666667)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 165.86075901985168, + "token_usage": { + "input_tokens": 24064, + "output_tokens": 4198, + "total_tokens": 28262, + "reasoning_tokens": 4127 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/company_in_a_box__employee_onboarding.json b/model_results/deepseek-v3-1-terminus-thinking/company_in_a_box__employee_onboarding.json new file mode 100644 index 00000000..3de7d411 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/company_in_a_box__employee_onboarding.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "company_in_a_box__employee_onboarding", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 334.27523612976074, + "token_usage": { + "input_tokens": 776693, + "output_tokens": 4201, + "total_tokens": 780894, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1877.3445603847504, + "token_usage": { + "input_tokens": 166115, + "output_tokens": 42539, + "total_tokens": 208654, + "reasoning_tokens": 41318 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 371.6750223636627, + "token_usage": { + "input_tokens": 827869, + "output_tokens": 4746, + "total_tokens": 832615, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 293.0011365413666, + "token_usage": { + "input_tokens": 456566, + "output_tokens": 3925, + "total_tokens": 460491, + "reasoning_tokens": 230 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/company_in_a_box__goals_restructure.json b/model_results/deepseek-v3-1-terminus-thinking/company_in_a_box__goals_restructure.json new file mode 100644 index 00000000..620b764b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/company_in_a_box__goals_restructure.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "company_in_a_box__goals_restructure", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 323.6598677635193, + "token_usage": { + "input_tokens": 765535, + "output_tokens": 2642, + "total_tokens": 768177, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 2635.1523876190186, + "token_usage": { + "input_tokens": 300996, + "output_tokens": 58621, + "total_tokens": 359617, + "reasoning_tokens": 55900 + }, + "turn_count": 13 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 263.1151399612427, + "token_usage": { + "input_tokens": 589729, + "output_tokens": 2378, + "total_tokens": 592107, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 423.7647581100464, + "token_usage": { + "input_tokens": 84801, + "output_tokens": 11370, + "total_tokens": 96171, + "reasoning_tokens": 10988 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/company_in_a_box__quarterly_review_dashboard.json b/model_results/deepseek-v3-1-terminus-thinking/company_in_a_box__quarterly_review_dashboard.json new file mode 100644 index 00000000..281c3b60 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/company_in_a_box__quarterly_review_dashboard.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "company_in_a_box__quarterly_review_dashboard", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 491.57786679267883, + "token_usage": { + "input_tokens": 1646630, + "output_tokens": 4840, + "total_tokens": 1651470, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 3055.5822019577026, + "token_usage": { + "input_tokens": 427190, + "output_tokens": 65443, + "total_tokens": 492633, + "reasoning_tokens": 61266 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": "Extra data: line 1 column 576 (char 575)", + "execution_time": 135.1262640953064, + "token_usage": { + "input_tokens": 298666, + "output_tokens": 1260, + "total_tokens": 299926, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 2390.3604912757874, + "token_usage": { + "input_tokens": 498080, + "output_tokens": 63478, + "total_tokens": 561558, + "reasoning_tokens": 55685 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/computer_science_student_dashboard__code_snippets_go.json b/model_results/deepseek-v3-1-terminus-thinking/computer_science_student_dashboard__code_snippets_go.json new file mode 100644 index 00000000..487d7c3e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/computer_science_student_dashboard__code_snippets_go.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "computer_science_student_dashboard__code_snippets_go", + "runs": { + "run-1": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 1187.0391972064972, + "token_usage": { + "input_tokens": 2858198, + "output_tokens": 15258, + "total_tokens": 2873456, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 1056.1947813034058, + "token_usage": { + "input_tokens": 150854, + "output_tokens": 24738, + "total_tokens": 175592, + "reasoning_tokens": 23142 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 204.9584858417511, + "token_usage": { + "input_tokens": 275104, + "output_tokens": 2642, + "total_tokens": 277746, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 817.3883492946625, + "token_usage": { + "input_tokens": 90491, + "output_tokens": 22676, + "total_tokens": 113167, + "reasoning_tokens": 21364 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/computer_science_student_dashboard__courses_internships_relation.json b/model_results/deepseek-v3-1-terminus-thinking/computer_science_student_dashboard__courses_internships_relation.json new file mode 100644 index 00000000..d4aa829f --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/computer_science_student_dashboard__courses_internships_relation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "computer_science_student_dashboard__courses_internships_relation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 277.7262635231018, + "token_usage": { + "input_tokens": 618412, + "output_tokens": 2972, + "total_tokens": 621384, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 2886.3092391490936, + "token_usage": { + "input_tokens": 237041, + "output_tokens": 61844, + "total_tokens": 298885, + "reasoning_tokens": 57715 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": "Extra data: line 1 column 483 (char 482)", + "execution_time": 118.15512609481812, + "token_usage": { + "input_tokens": 138966, + "output_tokens": 1347, + "total_tokens": 140313, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1408.530945777893, + "token_usage": { + "input_tokens": 157236, + "output_tokens": 38864, + "total_tokens": 196100, + "reasoning_tokens": 36770 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/computer_science_student_dashboard__study_session_tracker.json b/model_results/deepseek-v3-1-terminus-thinking/computer_science_student_dashboard__study_session_tracker.json new file mode 100644 index 00000000..f108e276 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/computer_science_student_dashboard__study_session_tracker.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "computer_science_student_dashboard__study_session_tracker", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 101.82571601867676, + "token_usage": { + "input_tokens": 130432, + "output_tokens": 1098, + "total_tokens": 131530, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1246.9643194675446, + "token_usage": { + "input_tokens": 152717, + "output_tokens": 26813, + "total_tokens": 179530, + "reasoning_tokens": 25489 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 120.59121704101562, + "token_usage": { + "input_tokens": 136996, + "output_tokens": 1293, + "total_tokens": 138289, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 650.3801589012146, + "token_usage": { + "input_tokens": 116499, + "output_tokens": 16871, + "total_tokens": 133370, + "reasoning_tokens": 15726 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/desktop__music_report.json b/model_results/deepseek-v3-1-terminus-thinking/desktop__music_report.json new file mode 100644 index 00000000..d44df0d7 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/desktop__music_report.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "desktop__music_report", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 247.4196126461029, + "token_usage": { + "input_tokens": 36556, + "output_tokens": 5128, + "total_tokens": 41684, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 173.90516328811646, + "token_usage": { + "input_tokens": 44065, + "output_tokens": 3045, + "total_tokens": 47110, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 614.7150180339813, + "token_usage": { + "input_tokens": 36101, + "output_tokens": 15219, + "total_tokens": 51320, + "reasoning_tokens": 14477 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 87.21180248260498, + "token_usage": { + "input_tokens": 9918, + "output_tokens": 1871, + "total_tokens": 11789, + "reasoning_tokens": 1772 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/desktop__project_management.json b/model_results/deepseek-v3-1-terminus-thinking/desktop__project_management.json new file mode 100644 index 00000000..4e30e4f8 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/desktop__project_management.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "desktop__project_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 340.08776688575745, + "token_usage": { + "input_tokens": 283339, + "output_tokens": 3971, + "total_tokens": 287310, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 405.20643281936646, + "token_usage": { + "input_tokens": 413338, + "output_tokens": 4891, + "total_tokens": 418229, + "reasoning_tokens": 0 + }, + "turn_count": 46 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 983.6114149093628, + "token_usage": { + "input_tokens": 65895, + "output_tokens": 22154, + "total_tokens": 88049, + "reasoning_tokens": 19806 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": "Extra data: line 1 column 1309 (char 1308)", + "execution_time": 1229.6339173316956, + "token_usage": { + "input_tokens": 99460, + "output_tokens": 32049, + "total_tokens": 131509, + "reasoning_tokens": 27719 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/desktop__timeline_extraction.json b/model_results/deepseek-v3-1-terminus-thinking/desktop__timeline_extraction.json new file mode 100644 index 00000000..0470ec25 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/desktop__timeline_extraction.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "desktop__timeline_extraction", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 129.97008752822876, + "token_usage": { + "input_tokens": 77219, + "output_tokens": 2438, + "total_tokens": 79657, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 1060.0679953098297, + "token_usage": { + "input_tokens": 126262, + "output_tokens": 28481, + "total_tokens": 154743, + "reasoning_tokens": 24102 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 1507.707370042801, + "token_usage": { + "input_tokens": 96849, + "output_tokens": 36846, + "total_tokens": 133695, + "reasoning_tokens": 33514 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1462.4173436164856, + "token_usage": { + "input_tokens": 80330, + "output_tokens": 40539, + "total_tokens": 120869, + "reasoning_tokens": 37710 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/desktop_template__budget_computation.json b/model_results/deepseek-v3-1-terminus-thinking/desktop_template__budget_computation.json new file mode 100644 index 00000000..d1fa7a65 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/desktop_template__budget_computation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "desktop_template__budget_computation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 113.12025308609009, + "token_usage": { + "input_tokens": 49961, + "output_tokens": 1778, + "total_tokens": 51739, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 332.6777753829956, + "token_usage": { + "input_tokens": 17654, + "output_tokens": 9258, + "total_tokens": 26912, + "reasoning_tokens": 8407 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 621.1506094932556, + "token_usage": { + "input_tokens": 16739, + "output_tokens": 14011, + "total_tokens": 30750, + "reasoning_tokens": 13644 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 787.7252607345581, + "token_usage": { + "input_tokens": 25612, + "output_tokens": 21529, + "total_tokens": 47141, + "reasoning_tokens": 20308 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/desktop_template__contact_information.json b/model_results/deepseek-v3-1-terminus-thinking/desktop_template__contact_information.json new file mode 100644 index 00000000..a85ddd11 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/desktop_template__contact_information.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "desktop_template__contact_information", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 146.865305185318, + "token_usage": { + "input_tokens": 73597, + "output_tokens": 2268, + "total_tokens": 75865, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 674.874340057373, + "token_usage": { + "input_tokens": 30080, + "output_tokens": 17997, + "total_tokens": 48077, + "reasoning_tokens": 16458 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 701.8135657310486, + "token_usage": { + "input_tokens": 37972, + "output_tokens": 16458, + "total_tokens": 54430, + "reasoning_tokens": 14966 + }, + "turn_count": 7 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 907.022073507309, + "token_usage": { + "input_tokens": 52498, + "output_tokens": 24754, + "total_tokens": 77252, + "reasoning_tokens": 23124 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/desktop_template__file_arrangement.json b/model_results/deepseek-v3-1-terminus-thinking/desktop_template__file_arrangement.json new file mode 100644 index 00000000..05eb9d80 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/desktop_template__file_arrangement.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "desktop_template__file_arrangement", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 436.8873176574707, + "token_usage": { + "input_tokens": 504678, + "output_tokens": 4914, + "total_tokens": 509592, + "reasoning_tokens": 0 + }, + "turn_count": 58 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1007.7418031692505, + "token_usage": { + "input_tokens": 80103, + "output_tokens": 27330, + "total_tokens": 107433, + "reasoning_tokens": 23666 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1110.845541715622, + "token_usage": { + "input_tokens": 88256, + "output_tokens": 26333, + "total_tokens": 114589, + "reasoning_tokens": 22165 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1315.487462759018, + "token_usage": { + "input_tokens": 80741, + "output_tokens": 36740, + "total_tokens": 117481, + "reasoning_tokens": 32638 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/dvdrental__customer_analysis_fix.json b/model_results/deepseek-v3-1-terminus-thinking/dvdrental__customer_analysis_fix.json new file mode 100644 index 00000000..5a38c370 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/dvdrental__customer_analysis_fix.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "dvdrental__customer_analysis_fix", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 615.4603867530823, + "token_usage": { + "input_tokens": 562080, + "output_tokens": 9415, + "total_tokens": 571495, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 202509261706047468401524573329)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 436.2656843662262, + "token_usage": { + "input_tokens": 112603, + "output_tokens": 2311, + "total_tokens": 114914, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 375.3437707424164, + "token_usage": { + "input_tokens": 412950, + "output_tokens": 6024, + "total_tokens": 418974, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 77.14166474342346, + "token_usage": { + "input_tokens": 3652, + "output_tokens": 1811, + "total_tokens": 5463, + "reasoning_tokens": 1780 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/dvdrental__customer_analytics_optimization.json b/model_results/deepseek-v3-1-terminus-thinking/dvdrental__customer_analytics_optimization.json new file mode 100644 index 00000000..91febf75 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/dvdrental__customer_analytics_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "dvdrental__customer_analytics_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 300.92593359947205, + "token_usage": { + "input_tokens": 171637, + "output_tokens": 4732, + "total_tokens": 176369, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 270.64186120033264, + "token_usage": { + "input_tokens": 190382, + "output_tokens": 4652, + "total_tokens": 195034, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 247.33644652366638, + "token_usage": { + "input_tokens": 186229, + "output_tokens": 3891, + "total_tokens": 190120, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 210.79088759422302, + "token_usage": { + "input_tokens": 12386, + "output_tokens": 5603, + "total_tokens": 17989, + "reasoning_tokens": 4398 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/dvdrental__film_inventory_management.json b/model_results/deepseek-v3-1-terminus-thinking/dvdrental__film_inventory_management.json new file mode 100644 index 00000000..8c2d99f3 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/dvdrental__film_inventory_management.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "dvdrental__film_inventory_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 265.45637702941895, + "token_usage": { + "input_tokens": 299262, + "output_tokens": 2949, + "total_tokens": 302211, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 264.9316096305847, + "token_usage": { + "input_tokens": 385200, + "output_tokens": 3164, + "total_tokens": 388364, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 224.0705850124359, + "token_usage": { + "input_tokens": 420685, + "output_tokens": 2560, + "total_tokens": 423245, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 495.70731568336487, + "token_usage": { + "input_tokens": 11750, + "output_tokens": 13154, + "total_tokens": 24904, + "reasoning_tokens": 11981 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/easyr1__advanced_branch_strategy.json b/model_results/deepseek-v3-1-terminus-thinking/easyr1__advanced_branch_strategy.json new file mode 100644 index 00000000..d44d0d05 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/easyr1__advanced_branch_strategy.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "easyr1__advanced_branch_strategy", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 133.41261291503906, + "token_usage": { + "input_tokens": 236001, + "output_tokens": 1385, + "total_tokens": 237386, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 657.4795656204224, + "token_usage": { + "input_tokens": 206206, + "output_tokens": 16956, + "total_tokens": 223162, + "reasoning_tokens": 15853 + }, + "turn_count": 8 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 969.739565372467, + "token_usage": { + "input_tokens": 281358, + "output_tokens": 20893, + "total_tokens": 302251, + "reasoning_tokens": 19803 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 43.79906630516052, + "token_usage": { + "input_tokens": 23930, + "output_tokens": 1206, + "total_tokens": 25136, + "reasoning_tokens": 315 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/easyr1__config_parameter_audit.json b/model_results/deepseek-v3-1-terminus-thinking/easyr1__config_parameter_audit.json new file mode 100644 index 00000000..1cefd925 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/easyr1__config_parameter_audit.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "easyr1__config_parameter_audit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 313.7692756652832, + "token_usage": { + "input_tokens": 1119193, + "output_tokens": 2255, + "total_tokens": 1121448, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 983.0928440093994, + "token_usage": { + "input_tokens": 257270, + "output_tokens": 24271, + "total_tokens": 281541, + "reasoning_tokens": 23047 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 789.3945915699005, + "token_usage": { + "input_tokens": 304602, + "output_tokens": 16554, + "total_tokens": 321156, + "reasoning_tokens": 16101 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 716.6567142009735, + "token_usage": { + "input_tokens": 287608, + "output_tokens": 19051, + "total_tokens": 306659, + "reasoning_tokens": 18659 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/easyr1__performance_regression_investigation.json b/model_results/deepseek-v3-1-terminus-thinking/easyr1__performance_regression_investigation.json new file mode 100644 index 00000000..4011255c --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/easyr1__performance_regression_investigation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "easyr1__performance_regression_investigation", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 190.212881565094, + "token_usage": { + "input_tokens": 292344, + "output_tokens": 2611, + "total_tokens": 294955, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 508.85288095474243, + "token_usage": { + "input_tokens": 90096, + "output_tokens": 12249, + "total_tokens": 102345, + "reasoning_tokens": 10191 + }, + "turn_count": 3 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 680.2623610496521, + "token_usage": { + "input_tokens": 129262, + "output_tokens": 15538, + "total_tokens": 144800, + "reasoning_tokens": 13387 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 478.04176664352417, + "token_usage": { + "input_tokens": 109780, + "output_tokens": 12801, + "total_tokens": 122581, + "reasoning_tokens": 11140 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/easyr1__qwen3_issue_management.json b/model_results/deepseek-v3-1-terminus-thinking/easyr1__qwen3_issue_management.json new file mode 100644 index 00000000..f022d795 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/easyr1__qwen3_issue_management.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "easyr1__qwen3_issue_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 133.20124340057373, + "token_usage": { + "input_tokens": 365355, + "output_tokens": 1200, + "total_tokens": 366555, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 253.49759078025818, + "token_usage": { + "input_tokens": 75866, + "output_tokens": 5494, + "total_tokens": 81360, + "reasoning_tokens": 4984 + }, + "turn_count": 3 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 778.1814601421356, + "token_usage": { + "input_tokens": 222374, + "output_tokens": 17335, + "total_tokens": 239709, + "reasoning_tokens": 16261 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 433.4517014026642, + "token_usage": { + "input_tokens": 164397, + "output_tokens": 11797, + "total_tokens": 176194, + "reasoning_tokens": 11095 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/employees__employee_demographics_report.json b/model_results/deepseek-v3-1-terminus-thinking/employees__employee_demographics_report.json new file mode 100644 index 00000000..3ed00246 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/employees__employee_demographics_report.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "employees__employee_demographics_report", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 373.7859036922455, + "token_usage": { + "input_tokens": 271794, + "output_tokens": 4959, + "total_tokens": 276753, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 345.913902759552, + "token_usage": { + "input_tokens": 232934, + "output_tokens": 5260, + "total_tokens": 238194, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 249.38017582893372, + "token_usage": { + "input_tokens": 177824, + "output_tokens": 3398, + "total_tokens": 181222, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 707.6514430046082, + "token_usage": { + "input_tokens": 60163, + "output_tokens": 18420, + "total_tokens": 78583, + "reasoning_tokens": 16586 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/employees__employee_performance_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/employees__employee_performance_analysis.json new file mode 100644 index 00000000..eafd693a --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/employees__employee_performance_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "employees__employee_performance_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 219.51662135124207, + "token_usage": { + "input_tokens": 132930, + "output_tokens": 2862, + "total_tokens": 135792, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 408.2539646625519, + "token_usage": { + "input_tokens": 280076, + "output_tokens": 6444, + "total_tokens": 286520, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 177.42112493515015, + "token_usage": { + "input_tokens": 119897, + "output_tokens": 2208, + "total_tokens": 122105, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 174.39799976348877, + "token_usage": { + "input_tokens": 24302, + "output_tokens": 3950, + "total_tokens": 28252, + "reasoning_tokens": 3695 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/employees__employee_project_tracking.json b/model_results/deepseek-v3-1-terminus-thinking/employees__employee_project_tracking.json new file mode 100644 index 00000000..c940a8d7 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/employees__employee_project_tracking.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "employees__employee_project_tracking", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 272.468674659729, + "token_usage": { + "input_tokens": 251023, + "output_tokens": 2924, + "total_tokens": 253947, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 281.83456468582153, + "token_usage": { + "input_tokens": 234144, + "output_tokens": 3163, + "total_tokens": 237307, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092621205390995004792915453)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 278.30049180984497, + "token_usage": { + "input_tokens": 109790, + "output_tokens": 985, + "total_tokens": 110775, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 291.89334869384766, + "token_usage": { + "input_tokens": 21612, + "output_tokens": 7253, + "total_tokens": 28865, + "reasoning_tokens": 7064 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/employees__employee_retention_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/employees__employee_retention_analysis.json new file mode 100644 index 00000000..79daf61a --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/employees__employee_retention_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "employees__employee_retention_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 306.8211603164673, + "token_usage": { + "input_tokens": 309162, + "output_tokens": 3581, + "total_tokens": 312743, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 322.8718523979187, + "token_usage": { + "input_tokens": 284053, + "output_tokens": 5027, + "total_tokens": 289080, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 237.07192850112915, + "token_usage": { + "input_tokens": 170293, + "output_tokens": 3422, + "total_tokens": 173715, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 105.83524990081787, + "token_usage": { + "input_tokens": 19309, + "output_tokens": 2355, + "total_tokens": 21664, + "reasoning_tokens": 2187 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/employees__executive_dashboard_automation.json b/model_results/deepseek-v3-1-terminus-thinking/employees__executive_dashboard_automation.json new file mode 100644 index 00000000..4e192397 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/employees__executive_dashboard_automation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "employees__executive_dashboard_automation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 307.86621141433716, + "token_usage": { + "input_tokens": 275964, + "output_tokens": 4327, + "total_tokens": 280291, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 305.2025291919708, + "token_usage": { + "input_tokens": 311463, + "output_tokens": 4370, + "total_tokens": 315833, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1015.3421957492828, + "token_usage": { + "input_tokens": 209079, + "output_tokens": 27140, + "total_tokens": 236219, + "reasoning_tokens": 23493 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": "Extra data: line 1 column 499 (char 498)", + "execution_time": 640.9347364902496, + "token_usage": { + "input_tokens": 21215, + "output_tokens": 16921, + "total_tokens": 38136, + "reasoning_tokens": 13951 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/employees__management_structure_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/employees__management_structure_analysis.json new file mode 100644 index 00000000..9c0d4d89 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/employees__management_structure_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "employees__management_structure_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 237.82022404670715, + "token_usage": { + "input_tokens": 227506, + "output_tokens": 3184, + "total_tokens": 230690, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 249.29724287986755, + "token_usage": { + "input_tokens": 195473, + "output_tokens": 3643, + "total_tokens": 199116, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 795.6358561515808, + "token_usage": { + "input_tokens": 61521, + "output_tokens": 21387, + "total_tokens": 82908, + "reasoning_tokens": 18882 + }, + "turn_count": 10 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 438.61567544937134, + "token_usage": { + "input_tokens": 38820, + "output_tokens": 10377, + "total_tokens": 49197, + "reasoning_tokens": 8762 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/eval_web__cloudflare_turnstile_challenge.json b/model_results/deepseek-v3-1-terminus-thinking/eval_web__cloudflare_turnstile_challenge.json new file mode 100644 index 00000000..b76ff66b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/eval_web__cloudflare_turnstile_challenge.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "eval_web__cloudflare_turnstile_challenge", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 569.7963891029358, + "token_usage": { + "input_tokens": 996745, + "output_tokens": 8078, + "total_tokens": 1004823, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 505.6805808544159, + "token_usage": { + "input_tokens": 655707, + "output_tokens": 8209, + "total_tokens": 663916, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-3": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 14 (char 13)", + "execution_time": 409.2838089466095, + "token_usage": { + "input_tokens": 258539, + "output_tokens": 7988, + "total_tokens": 266527, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 424.75878071784973, + "token_usage": { + "input_tokens": 351222, + "output_tokens": 6008, + "total_tokens": 357230, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/eval_web__extraction_table.json b/model_results/deepseek-v3-1-terminus-thinking/eval_web__extraction_table.json new file mode 100644 index 00000000..8a513c34 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/eval_web__extraction_table.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "eval_web__extraction_table", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 428.7061369419098, + "token_usage": { + "input_tokens": 1009113, + "output_tokens": 6129, + "total_tokens": 1015242, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 325.54053139686584, + "token_usage": { + "input_tokens": 701525, + "output_tokens": 5203, + "total_tokens": 706728, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 352.2859034538269, + "token_usage": { + "input_tokens": 737270, + "output_tokens": 5313, + "total_tokens": 742583, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 423.85006046295166, + "token_usage": { + "input_tokens": 714910, + "output_tokens": 5815, + "total_tokens": 720725, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/file_context__duplicates_searching.json b/model_results/deepseek-v3-1-terminus-thinking/file_context__duplicates_searching.json new file mode 100644 index 00000000..2af5a64e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/file_context__duplicates_searching.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "file_context__duplicates_searching", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 153.30653977394104, + "token_usage": { + "input_tokens": 82054, + "output_tokens": 2291, + "total_tokens": 84345, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 624.7218894958496, + "token_usage": { + "input_tokens": 39607, + "output_tokens": 17869, + "total_tokens": 57476, + "reasoning_tokens": 15423 + }, + "turn_count": 7 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 643.4807195663452, + "token_usage": { + "input_tokens": 33792, + "output_tokens": 16341, + "total_tokens": 50133, + "reasoning_tokens": 13667 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 852.3943758010864, + "token_usage": { + "input_tokens": 32832, + "output_tokens": 17130, + "total_tokens": 49962, + "reasoning_tokens": 15242 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/file_context__file_merging.json b/model_results/deepseek-v3-1-terminus-thinking/file_context__file_merging.json new file mode 100644 index 00000000..8c77b5af --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/file_context__file_merging.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "file_context__file_merging", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 78.707355260849, + "token_usage": { + "input_tokens": 20206, + "output_tokens": 1340, + "total_tokens": 21546, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 559.9810602664948, + "token_usage": { + "input_tokens": 23358, + "output_tokens": 15322, + "total_tokens": 38680, + "reasoning_tokens": 14273 + }, + "turn_count": 6 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 531.8824737071991, + "token_usage": { + "input_tokens": 19174, + "output_tokens": 14058, + "total_tokens": 33232, + "reasoning_tokens": 13106 + }, + "turn_count": 5 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 764.0157697200775, + "token_usage": { + "input_tokens": 27109, + "output_tokens": 19977, + "total_tokens": 47086, + "reasoning_tokens": 18842 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/file_context__file_splitting.json b/model_results/deepseek-v3-1-terminus-thinking/file_context__file_splitting.json new file mode 100644 index 00000000..892ff520 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/file_context__file_splitting.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "file_context__file_splitting", + "runs": { + "run-1": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 1337 (char 1336)", + "execution_time": 52.51876521110535, + "token_usage": { + "input_tokens": 16014, + "output_tokens": 623, + "total_tokens": 16637, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 2419.200646162033, + "token_usage": { + "input_tokens": 32580, + "output_tokens": 33562, + "total_tokens": 66142, + "reasoning_tokens": 32353 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": "Filesystem Operation timed out", + "execution_time": 2839.5280299186707, + "token_usage": { + "input_tokens": 17345, + "output_tokens": 28203, + "total_tokens": 45548, + "reasoning_tokens": 25739 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": "Execution timed out after 3600 seconds", + "execution_time": 3600.1284716129303, + "token_usage": { + "input_tokens": 47562, + "output_tokens": 61449, + "total_tokens": 109011, + "reasoning_tokens": 58831 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/file_context__pattern_matching.json b/model_results/deepseek-v3-1-terminus-thinking/file_context__pattern_matching.json new file mode 100644 index 00000000..ab2595cc --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/file_context__pattern_matching.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "file_context__pattern_matching", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 105.45835661888123, + "token_usage": { + "input_tokens": 26734, + "output_tokens": 1853, + "total_tokens": 28587, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 1728.1030645370483, + "token_usage": { + "input_tokens": 31823, + "output_tokens": 31964, + "total_tokens": 63787, + "reasoning_tokens": 30866 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": "Extra data: line 1 column 147 (char 146)", + "execution_time": 883.2345929145813, + "token_usage": { + "input_tokens": 24442, + "output_tokens": 23258, + "total_tokens": 47700, + "reasoning_tokens": 22192 + }, + "turn_count": 5 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 3167.353559732437, + "token_usage": { + "input_tokens": 69485, + "output_tokens": 91934, + "total_tokens": 161419, + "reasoning_tokens": 90362 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/file_context__uppercase.json b/model_results/deepseek-v3-1-terminus-thinking/file_context__uppercase.json new file mode 100644 index 00000000..9a8caf09 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/file_context__uppercase.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "file_context__uppercase", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 147.84447693824768, + "token_usage": { + "input_tokens": 80409, + "output_tokens": 1927, + "total_tokens": 82336, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 699.920937538147, + "token_usage": { + "input_tokens": 28712, + "output_tokens": 19449, + "total_tokens": 48161, + "reasoning_tokens": 17523 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 980.858642578125, + "token_usage": { + "input_tokens": 28715, + "output_tokens": 25206, + "total_tokens": 53921, + "reasoning_tokens": 23197 + }, + "turn_count": 7 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 617.2144229412079, + "token_usage": { + "input_tokens": 17251, + "output_tokens": 17797, + "total_tokens": 35048, + "reasoning_tokens": 15882 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/file_property__size_classification.json b/model_results/deepseek-v3-1-terminus-thinking/file_property__size_classification.json new file mode 100644 index 00000000..f2ec20ff --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/file_property__size_classification.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "file_property__size_classification", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 118.75910472869873, + "token_usage": { + "input_tokens": 35182, + "output_tokens": 2097, + "total_tokens": 37279, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 346.4201979637146, + "token_usage": { + "input_tokens": 22003, + "output_tokens": 8690, + "total_tokens": 30693, + "reasoning_tokens": 7161 + }, + "turn_count": 5 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 483.41164898872375, + "token_usage": { + "input_tokens": 27939, + "output_tokens": 11448, + "total_tokens": 39387, + "reasoning_tokens": 9950 + }, + "turn_count": 6 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 351.00888657569885, + "token_usage": { + "input_tokens": 33871, + "output_tokens": 8957, + "total_tokens": 42828, + "reasoning_tokens": 7094 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/file_property__time_classification.json b/model_results/deepseek-v3-1-terminus-thinking/file_property__time_classification.json new file mode 100644 index 00000000..2a2a5fa9 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/file_property__time_classification.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "file_property__time_classification", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 177.63192105293274, + "token_usage": { + "input_tokens": 107155, + "output_tokens": 2069, + "total_tokens": 109224, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 800.0584149360657, + "token_usage": { + "input_tokens": 31385, + "output_tokens": 20574, + "total_tokens": 51959, + "reasoning_tokens": 17946 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 743.9570920467377, + "token_usage": { + "input_tokens": 30651, + "output_tokens": 18341, + "total_tokens": 48992, + "reasoning_tokens": 15712 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 627.0146262645721, + "token_usage": { + "input_tokens": 41123, + "output_tokens": 17698, + "total_tokens": 58821, + "reasoning_tokens": 14092 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/folder_structure__structure_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/folder_structure__structure_analysis.json new file mode 100644 index 00000000..814e8869 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/folder_structure__structure_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "folder_structure__structure_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 206.6289942264557, + "token_usage": { + "input_tokens": 400006, + "output_tokens": 2645, + "total_tokens": 402651, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": "Filesystem Operation timed out", + "execution_time": 3339.836974143982, + "token_usage": { + "input_tokens": 14293, + "output_tokens": 24081, + "total_tokens": 38374, + "reasoning_tokens": 23906 + }, + "turn_count": 3 + }, + "run-3": { + "success": false, + "error_message": "Execution timed out after 3600 seconds", + "execution_time": 3600.0835769176483, + "token_usage": { + "input_tokens": 39652, + "output_tokens": 44229, + "total_tokens": 83881, + "reasoning_tokens": 43921 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": "Filesystem Operation timed out", + "execution_time": 2901.658514738083, + "token_usage": { + "input_tokens": 47788, + "output_tokens": 31581, + "total_tokens": 79369, + "reasoning_tokens": 31052 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/folder_structure__structure_mirror.json b/model_results/deepseek-v3-1-terminus-thinking/folder_structure__structure_mirror.json new file mode 100644 index 00000000..c90abff0 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/folder_structure__structure_mirror.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "folder_structure__structure_mirror", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 603.8157112598419, + "token_usage": { + "input_tokens": 934563, + "output_tokens": 6244, + "total_tokens": 940807, + "reasoning_tokens": 0 + }, + "turn_count": 75 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 1907.7407739162445, + "token_usage": { + "input_tokens": 41848, + "output_tokens": 31035, + "total_tokens": 72883, + "reasoning_tokens": 28470 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 2274.7533860206604, + "token_usage": { + "input_tokens": 71924, + "output_tokens": 58250, + "total_tokens": 130174, + "reasoning_tokens": 53564 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1513.5469272136688, + "token_usage": { + "input_tokens": 54984, + "output_tokens": 43127, + "total_tokens": 98111, + "reasoning_tokens": 40233 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/harmony__fix_conflict.json b/model_results/deepseek-v3-1-terminus-thinking/harmony__fix_conflict.json new file mode 100644 index 00000000..874f40d1 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/harmony__fix_conflict.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "harmony__fix_conflict", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 331.02082896232605, + "token_usage": { + "input_tokens": 1614626, + "output_tokens": 2797, + "total_tokens": 1617423, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 641.3220250606537, + "token_usage": { + "input_tokens": 493963, + "output_tokens": 5939, + "total_tokens": 499902, + "reasoning_tokens": 5520 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 346.60584688186646, + "token_usage": { + "input_tokens": 257396, + "output_tokens": 5998, + "total_tokens": 263394, + "reasoning_tokens": 5592 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 133.6654496192932, + "token_usage": { + "input_tokens": 141244, + "output_tokens": 3187, + "total_tokens": 144431, + "reasoning_tokens": 3047 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/harmony__issue_pr_commit_workflow.json b/model_results/deepseek-v3-1-terminus-thinking/harmony__issue_pr_commit_workflow.json new file mode 100644 index 00000000..504690b2 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/harmony__issue_pr_commit_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "harmony__issue_pr_commit_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 157.60750436782837, + "token_usage": { + "input_tokens": 191573, + "output_tokens": 2282, + "total_tokens": 193855, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 185.31686425209045, + "token_usage": { + "input_tokens": 24220, + "output_tokens": 4053, + "total_tokens": 28273, + "reasoning_tokens": 3996 + }, + "turn_count": 1 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 546.4371287822723, + "token_usage": { + "input_tokens": 109899, + "output_tokens": 11975, + "total_tokens": 121874, + "reasoning_tokens": 9271 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 587.9246520996094, + "token_usage": { + "input_tokens": 112335, + "output_tokens": 16781, + "total_tokens": 129116, + "reasoning_tokens": 13917 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/harmony__issue_tagging_pr_closure.json b/model_results/deepseek-v3-1-terminus-thinking/harmony__issue_tagging_pr_closure.json new file mode 100644 index 00000000..fa7dced5 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/harmony__issue_tagging_pr_closure.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "harmony__issue_tagging_pr_closure", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 196.77281999588013, + "token_usage": { + "input_tokens": 250367, + "output_tokens": 2665, + "total_tokens": 253032, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 883.2635576725006, + "token_usage": { + "input_tokens": 236819, + "output_tokens": 17714, + "total_tokens": 254533, + "reasoning_tokens": 15680 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 69.26510071754456, + "token_usage": { + "input_tokens": 24341, + "output_tokens": 1566, + "total_tokens": 25907, + "reasoning_tokens": 1239 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 303.78307580947876, + "token_usage": { + "input_tokens": 91207, + "output_tokens": 8048, + "total_tokens": 99255, + "reasoning_tokens": 5073 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/harmony__multi_branch_commit_aggregation.json b/model_results/deepseek-v3-1-terminus-thinking/harmony__multi_branch_commit_aggregation.json new file mode 100644 index 00000000..81a6d470 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/harmony__multi_branch_commit_aggregation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "harmony__multi_branch_commit_aggregation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 233.56022357940674, + "token_usage": { + "input_tokens": 309211, + "output_tokens": 3501, + "total_tokens": 312712, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1701.85706615448, + "token_usage": { + "input_tokens": 268995, + "output_tokens": 37542, + "total_tokens": 306537, + "reasoning_tokens": 35439 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": "Extra data: line 1 column 281 (char 280)", + "execution_time": 1543.3553655147552, + "token_usage": { + "input_tokens": 263603, + "output_tokens": 36653, + "total_tokens": 300256, + "reasoning_tokens": 34586 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1226.5491137504578, + "token_usage": { + "input_tokens": 179145, + "output_tokens": 35408, + "total_tokens": 214553, + "reasoning_tokens": 31524 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/harmony__release_management_workflow.json b/model_results/deepseek-v3-1-terminus-thinking/harmony__release_management_workflow.json new file mode 100644 index 00000000..78a2cc76 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/harmony__release_management_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "harmony__release_management_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 407.90374398231506, + "token_usage": { + "input_tokens": 1602791, + "output_tokens": 4474, + "total_tokens": 1607265, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1450.204171180725, + "token_usage": { + "input_tokens": 503960, + "output_tokens": 28106, + "total_tokens": 532066, + "reasoning_tokens": 26182 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 534 (char 533)", + "execution_time": 2195.718644618988, + "token_usage": { + "input_tokens": 550492, + "output_tokens": 47249, + "total_tokens": 597741, + "reasoning_tokens": 39114 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1330.544644355774, + "token_usage": { + "input_tokens": 915634, + "output_tokens": 34559, + "total_tokens": 950193, + "reasoning_tokens": 32835 + }, + "turn_count": 16 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/it_trouble_shooting_hub__asset_retirement_migration.json b/model_results/deepseek-v3-1-terminus-thinking/it_trouble_shooting_hub__asset_retirement_migration.json new file mode 100644 index 00000000..a51699e6 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/it_trouble_shooting_hub__asset_retirement_migration.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "it_trouble_shooting_hub__asset_retirement_migration", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 304.82218194007874, + "token_usage": { + "input_tokens": 958294, + "output_tokens": 3066, + "total_tokens": 961360, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 333.83636116981506, + "token_usage": { + "input_tokens": 6836, + "output_tokens": 8372, + "total_tokens": 15208, + "reasoning_tokens": 8310 + }, + "turn_count": 1 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 177.39050602912903, + "token_usage": { + "input_tokens": 355670, + "output_tokens": 1783, + "total_tokens": 357453, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1051.4181864261627, + "token_usage": { + "input_tokens": 30831, + "output_tokens": 9553, + "total_tokens": 40384, + "reasoning_tokens": 9388 + }, + "turn_count": 2 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/it_trouble_shooting_hub__security_audit_ticket.json b/model_results/deepseek-v3-1-terminus-thinking/it_trouble_shooting_hub__security_audit_ticket.json new file mode 100644 index 00000000..d3a9f4dd --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/it_trouble_shooting_hub__security_audit_ticket.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "it_trouble_shooting_hub__security_audit_ticket", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 222.0452995300293, + "token_usage": { + "input_tokens": 536938, + "output_tokens": 2505, + "total_tokens": 539443, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 1014.8265445232391, + "token_usage": { + "input_tokens": 209350, + "output_tokens": 21222, + "total_tokens": 230572, + "reasoning_tokens": 20029 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 1007.6195752620697, + "token_usage": { + "input_tokens": 4683674, + "output_tokens": 9331, + "total_tokens": 4693005, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 990.9929640293121, + "token_usage": { + "input_tokens": 288830, + "output_tokens": 26772, + "total_tokens": 315602, + "reasoning_tokens": 25457 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/it_trouble_shooting_hub__verification_expired_update.json b/model_results/deepseek-v3-1-terminus-thinking/it_trouble_shooting_hub__verification_expired_update.json new file mode 100644 index 00000000..62aa1664 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/it_trouble_shooting_hub__verification_expired_update.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "it_trouble_shooting_hub__verification_expired_update", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 212.68869137763977, + "token_usage": { + "input_tokens": 461583, + "output_tokens": 2347, + "total_tokens": 463930, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 2508.9891328811646, + "token_usage": { + "input_tokens": 390069, + "output_tokens": 51477, + "total_tokens": 441546, + "reasoning_tokens": 49945 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 326.9920198917389, + "token_usage": { + "input_tokens": 1327503, + "output_tokens": 2938, + "total_tokens": 1330441, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 398.0065813064575, + "token_usage": { + "input_tokens": 6688, + "output_tokens": 12021, + "total_tokens": 18709, + "reasoning_tokens": 11959 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__daily_itinerary_overview.json b/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__daily_itinerary_overview.json new file mode 100644 index 00000000..126a89cb --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__daily_itinerary_overview.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "japan_travel_planner__daily_itinerary_overview", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 247.0055320262909, + "token_usage": { + "input_tokens": 682445, + "output_tokens": 3097, + "total_tokens": 685542, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1449.372210264206, + "token_usage": { + "input_tokens": 151449, + "output_tokens": 35684, + "total_tokens": 187133, + "reasoning_tokens": 30556 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092616214214254787301729991)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 526.1702470779419, + "token_usage": { + "input_tokens": 541614, + "output_tokens": 2567, + "total_tokens": 544181, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 929.0850973129272, + "token_usage": { + "input_tokens": 102091, + "output_tokens": 26858, + "total_tokens": 128949, + "reasoning_tokens": 24100 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__packing_progress_summary.json b/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__packing_progress_summary.json new file mode 100644 index 00000000..b5d50a1e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__packing_progress_summary.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "japan_travel_planner__packing_progress_summary", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 296.34033703804016, + "token_usage": { + "input_tokens": 1145499, + "output_tokens": 1985, + "total_tokens": 1147484, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 202509270904031134727946899379)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1949.6066291332245, + "token_usage": { + "input_tokens": 372929, + "output_tokens": 40385, + "total_tokens": 413314, + "reasoning_tokens": 38253 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 254.55807900428772, + "token_usage": { + "input_tokens": 1012181, + "output_tokens": 1916, + "total_tokens": 1014097, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1101.5449602603912, + "token_usage": { + "input_tokens": 256167, + "output_tokens": 28183, + "total_tokens": 284350, + "reasoning_tokens": 26361 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__remove_osaka_itinerary.json b/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__remove_osaka_itinerary.json new file mode 100644 index 00000000..5f4c72b6 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__remove_osaka_itinerary.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "japan_travel_planner__remove_osaka_itinerary", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 78.14082646369934, + "token_usage": { + "input_tokens": 136775, + "output_tokens": 881, + "total_tokens": 137656, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 378.673992395401, + "token_usage": { + "input_tokens": 75380, + "output_tokens": 8564, + "total_tokens": 83944, + "reasoning_tokens": 7959 + }, + "turn_count": 4 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 128.21069312095642, + "token_usage": { + "input_tokens": 472884, + "output_tokens": 943, + "total_tokens": 473827, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 385.25209307670593, + "token_usage": { + "input_tokens": 79682, + "output_tokens": 9774, + "total_tokens": 89456, + "reasoning_tokens": 9112 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__restaurant_expenses_sync.json b/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__restaurant_expenses_sync.json new file mode 100644 index 00000000..10715536 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/japan_travel_planner__restaurant_expenses_sync.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "japan_travel_planner__restaurant_expenses_sync", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 230.64241123199463, + "token_usage": { + "input_tokens": 353236, + "output_tokens": 2874, + "total_tokens": 356110, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1191.5031871795654, + "token_usage": { + "input_tokens": 80571, + "output_tokens": 30284, + "total_tokens": 110855, + "reasoning_tokens": 28241 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 230.10843515396118, + "token_usage": { + "input_tokens": 364540, + "output_tokens": 2657, + "total_tokens": 367197, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 1865.1096251010895, + "token_usage": { + "input_tokens": 197560, + "output_tokens": 45201, + "total_tokens": 242761, + "reasoning_tokens": 40564 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/legal_document__dispute_review.json b/model_results/deepseek-v3-1-terminus-thinking/legal_document__dispute_review.json new file mode 100644 index 00000000..5f783cf3 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/legal_document__dispute_review.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "legal_document__dispute_review", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 76.4704864025116, + "token_usage": { + "input_tokens": 197095, + "output_tokens": 647, + "total_tokens": 197742, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 649.4551486968994, + "token_usage": { + "input_tokens": 163336, + "output_tokens": 14215, + "total_tokens": 177551, + "reasoning_tokens": 13394 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 205.48431158065796, + "token_usage": { + "input_tokens": 17237, + "output_tokens": 5036, + "total_tokens": 22273, + "reasoning_tokens": 4824 + }, + "turn_count": 5 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 518.650547504425, + "token_usage": { + "input_tokens": 163312, + "output_tokens": 12786, + "total_tokens": 176098, + "reasoning_tokens": 12258 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/legal_document__individual_comments.json b/model_results/deepseek-v3-1-terminus-thinking/legal_document__individual_comments.json new file mode 100644 index 00000000..def83a68 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/legal_document__individual_comments.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "legal_document__individual_comments", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 164.35221314430237, + "token_usage": { + "input_tokens": 260063, + "output_tokens": 2280, + "total_tokens": 262343, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 676.6143836975098, + "token_usage": { + "input_tokens": 179927, + "output_tokens": 15412, + "total_tokens": 195339, + "reasoning_tokens": 14694 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 3041.0798683166504, + "token_usage": { + "input_tokens": 944952, + "output_tokens": 61653, + "total_tokens": 1006605, + "reasoning_tokens": 59085 + }, + "turn_count": 16 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 390.39178919792175, + "token_usage": { + "input_tokens": 179904, + "output_tokens": 9799, + "total_tokens": 189703, + "reasoning_tokens": 9042 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/legal_document__solution_tracing.json b/model_results/deepseek-v3-1-terminus-thinking/legal_document__solution_tracing.json new file mode 100644 index 00000000..db588f90 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/legal_document__solution_tracing.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "legal_document__solution_tracing", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 130.7681760787964, + "token_usage": { + "input_tokens": 215375, + "output_tokens": 1409, + "total_tokens": 216784, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092702092173807556664147440)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 440.03814792633057, + "token_usage": { + "input_tokens": 18018, + "output_tokens": 3404, + "total_tokens": 21422, + "reasoning_tokens": 2864 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 100.01496195793152, + "token_usage": { + "input_tokens": 6747, + "output_tokens": 2365, + "total_tokens": 9112, + "reasoning_tokens": 2330 + }, + "turn_count": 2 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092723242643851501313162210)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 237.7526581287384, + "token_usage": { + "input_tokens": 14140, + "output_tokens": 4602, + "total_tokens": 18742, + "reasoning_tokens": 4110 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/lego__consistency_enforcement.json b/model_results/deepseek-v3-1-terminus-thinking/lego__consistency_enforcement.json new file mode 100644 index 00000000..1b0494b2 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/lego__consistency_enforcement.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "lego__consistency_enforcement", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 259.01617765426636, + "token_usage": { + "input_tokens": 722407, + "output_tokens": 3651, + "total_tokens": 726058, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 319.63316440582275, + "token_usage": { + "input_tokens": 1151155, + "output_tokens": 4392, + "total_tokens": 1155547, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 1277.2306110858917, + "token_usage": { + "input_tokens": 324866, + "output_tokens": 27303, + "total_tokens": 352169, + "reasoning_tokens": 23920 + }, + "turn_count": 10 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 532.2614402770996, + "token_usage": { + "input_tokens": 68391, + "output_tokens": 13364, + "total_tokens": 81755, + "reasoning_tokens": 12151 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/lego__database_security_policies.json b/model_results/deepseek-v3-1-terminus-thinking/lego__database_security_policies.json new file mode 100644 index 00000000..a6e768fd --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/lego__database_security_policies.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "lego__database_security_policies", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 300.08112144470215, + "token_usage": { + "input_tokens": 177379, + "output_tokens": 4401, + "total_tokens": 181780, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 539.3631503582001, + "token_usage": { + "input_tokens": 381895, + "output_tokens": 8235, + "total_tokens": 390130, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 929.1428561210632, + "token_usage": { + "input_tokens": 100425, + "output_tokens": 24177, + "total_tokens": 124602, + "reasoning_tokens": 21045 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 331.6807019710541, + "token_usage": { + "input_tokens": 10009, + "output_tokens": 7753, + "total_tokens": 17762, + "reasoning_tokens": 7677 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/lego__transactional_inventory_transfer.json b/model_results/deepseek-v3-1-terminus-thinking/lego__transactional_inventory_transfer.json new file mode 100644 index 00000000..29b57235 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/lego__transactional_inventory_transfer.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "lego__transactional_inventory_transfer", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 867.2886288166046, + "token_usage": { + "input_tokens": 2997259, + "output_tokens": 13824, + "total_tokens": 3011083, + "reasoning_tokens": 0 + }, + "turn_count": 50 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 711.7165093421936, + "token_usage": { + "input_tokens": 688995, + "output_tokens": 12298, + "total_tokens": 701293, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 122.4326663017273, + "token_usage": { + "input_tokens": 3125, + "output_tokens": 3010, + "total_tokens": 6135, + "reasoning_tokens": 603 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1007.8619937896729, + "token_usage": { + "input_tokens": 22228, + "output_tokens": 9828, + "total_tokens": 32056, + "reasoning_tokens": 7576 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__deployment_status_workflow.json b/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__deployment_status_workflow.json new file mode 100644 index 00000000..fcc065c9 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__deployment_status_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "mcpmark-cicd__deployment_status_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 266.72085332870483, + "token_usage": { + "input_tokens": 145178, + "output_tokens": 5114, + "total_tokens": 150292, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 685.8437578678131, + "token_usage": { + "input_tokens": 107018, + "output_tokens": 15088, + "total_tokens": 122106, + "reasoning_tokens": 12341 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 675.1347379684448, + "token_usage": { + "input_tokens": 162826, + "output_tokens": 15080, + "total_tokens": 177906, + "reasoning_tokens": 11477 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 493.03128576278687, + "token_usage": { + "input_tokens": 107142, + "output_tokens": 14013, + "total_tokens": 121155, + "reasoning_tokens": 11437 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__issue_management_workflow.json b/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__issue_management_workflow.json new file mode 100644 index 00000000..725e5f01 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__issue_management_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "mcpmark-cicd__issue_management_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 537.5137591362, + "token_usage": { + "input_tokens": 1028661, + "output_tokens": 7924, + "total_tokens": 1036585, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1096.5259656906128, + "token_usage": { + "input_tokens": 219938, + "output_tokens": 23213, + "total_tokens": 243151, + "reasoning_tokens": 17906 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1718.2038402557373, + "token_usage": { + "input_tokens": 338588, + "output_tokens": 21236, + "total_tokens": 359824, + "reasoning_tokens": 16802 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 791.8517625331879, + "token_usage": { + "input_tokens": 110326, + "output_tokens": 22859, + "total_tokens": 133185, + "reasoning_tokens": 18835 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__linting_ci_workflow.json b/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__linting_ci_workflow.json new file mode 100644 index 00000000..b05cfd19 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__linting_ci_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "mcpmark-cicd__linting_ci_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 119.7724871635437, + "token_usage": { + "input_tokens": 133512, + "output_tokens": 1688, + "total_tokens": 135200, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 379.9204249382019, + "token_usage": { + "input_tokens": 49738, + "output_tokens": 8380, + "total_tokens": 58118, + "reasoning_tokens": 7401 + }, + "turn_count": 2 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 786.3401265144348, + "token_usage": { + "input_tokens": 182005, + "output_tokens": 15134, + "total_tokens": 197139, + "reasoning_tokens": 13585 + }, + "turn_count": 7 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 462.70820474624634, + "token_usage": { + "input_tokens": 101168, + "output_tokens": 13098, + "total_tokens": 114266, + "reasoning_tokens": 11758 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__pr_automation_workflow.json b/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__pr_automation_workflow.json new file mode 100644 index 00000000..cf3d2aa2 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/mcpmark-cicd__pr_automation_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "mcpmark-cicd__pr_automation_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 273.5244388580322, + "token_usage": { + "input_tokens": 266175, + "output_tokens": 4311, + "total_tokens": 270486, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 417.759516954422, + "token_usage": { + "input_tokens": 108295, + "output_tokens": 8799, + "total_tokens": 117094, + "reasoning_tokens": 4313 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 880.5527496337891, + "token_usage": { + "input_tokens": 196076, + "output_tokens": 18662, + "total_tokens": 214738, + "reasoning_tokens": 16183 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 690.0449638366699, + "token_usage": { + "input_tokens": 127083, + "output_tokens": 20718, + "total_tokens": 147801, + "reasoning_tokens": 19357 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/missing-semester__assign_contributor_labels.json b/model_results/deepseek-v3-1-terminus-thinking/missing-semester__assign_contributor_labels.json new file mode 100644 index 00000000..7c091c9b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/missing-semester__assign_contributor_labels.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "missing-semester__assign_contributor_labels", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 253.2252287864685, + "token_usage": { + "input_tokens": 1266459, + "output_tokens": 1772, + "total_tokens": 1268231, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 2348.931410074234, + "token_usage": { + "input_tokens": 510937, + "output_tokens": 51406, + "total_tokens": 562343, + "reasoning_tokens": 50324 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 2197.3706357479095, + "token_usage": { + "input_tokens": 393220, + "output_tokens": 47749, + "total_tokens": 440969, + "reasoning_tokens": 45008 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 2324.812409877777, + "token_usage": { + "input_tokens": 548256, + "output_tokens": 45120, + "total_tokens": 593376, + "reasoning_tokens": 42697 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/missing-semester__find_legacy_name.json b/model_results/deepseek-v3-1-terminus-thinking/missing-semester__find_legacy_name.json new file mode 100644 index 00000000..252d4a21 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/missing-semester__find_legacy_name.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "missing-semester__find_legacy_name", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092620280339945617625908171)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 253.85763430595398, + "token_usage": { + "input_tokens": 1212872, + "output_tokens": 878, + "total_tokens": 1213750, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 488.5240752696991, + "token_usage": { + "input_tokens": 246380, + "output_tokens": 9651, + "total_tokens": 256031, + "reasoning_tokens": 9417 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092715155896474057277079855)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 689.0475895404816, + "token_usage": { + "input_tokens": 161943, + "output_tokens": 7204, + "total_tokens": 169147, + "reasoning_tokens": 6986 + }, + "turn_count": 3 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 195.33893036842346, + "token_usage": { + "input_tokens": 132351, + "output_tokens": 4721, + "total_tokens": 137072, + "reasoning_tokens": 4562 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/missing-semester__find_salient_file.json b/model_results/deepseek-v3-1-terminus-thinking/missing-semester__find_salient_file.json new file mode 100644 index 00000000..049452ef --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/missing-semester__find_salient_file.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "github", + "task": "missing-semester__find_salient_file", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 389.75921511650085, + "token_usage": { + "input_tokens": 1782919, + "output_tokens": 3457, + "total_tokens": 1786376, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + "run-2": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 62 (char 61)", + "execution_time": 649.1462063789368, + "token_usage": { + "input_tokens": 101856, + "output_tokens": 15399, + "total_tokens": 117255, + "reasoning_tokens": 11191 + }, + "turn_count": 3 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1574.7181437015533, + "token_usage": { + "input_tokens": 435629, + "output_tokens": 35580, + "total_tokens": 471209, + "reasoning_tokens": 32311 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 111 (char 110)", + "execution_time": 1646.8885293006897, + "token_usage": { + "input_tokens": 423388, + "output_tokens": 44367, + "total_tokens": 467755, + "reasoning_tokens": 38187 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/online_resume__layout_adjustment.json b/model_results/deepseek-v3-1-terminus-thinking/online_resume__layout_adjustment.json new file mode 100644 index 00000000..10c16c25 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/online_resume__layout_adjustment.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "online_resume__layout_adjustment", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 399.45585775375366, + "token_usage": { + "input_tokens": 1347645, + "output_tokens": 3756, + "total_tokens": 1351401, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 795.1876745223999, + "token_usage": { + "input_tokens": 154670, + "output_tokens": 19477, + "total_tokens": 174147, + "reasoning_tokens": 17564 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 343.291757106781, + "token_usage": { + "input_tokens": 1196089, + "output_tokens": 3083, + "total_tokens": 1199172, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 2117.443124771118, + "token_usage": { + "input_tokens": 416183, + "output_tokens": 42671, + "total_tokens": 458854, + "reasoning_tokens": 39603 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/online_resume__projects_section_update.json b/model_results/deepseek-v3-1-terminus-thinking/online_resume__projects_section_update.json new file mode 100644 index 00000000..ac8bde22 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/online_resume__projects_section_update.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "online_resume__projects_section_update", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 242.21766686439514, + "token_usage": { + "input_tokens": 699039, + "output_tokens": 2334, + "total_tokens": 701373, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 1428.3427770137787, + "token_usage": { + "input_tokens": 267372, + "output_tokens": 32971, + "total_tokens": 300343, + "reasoning_tokens": 31280 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 214.03604531288147, + "token_usage": { + "input_tokens": 567973, + "output_tokens": 2085, + "total_tokens": 570058, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1910.665857553482, + "token_usage": { + "input_tokens": 283888, + "output_tokens": 41540, + "total_tokens": 325428, + "reasoning_tokens": 39272 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/online_resume__skills_development_tracker.json b/model_results/deepseek-v3-1-terminus-thinking/online_resume__skills_development_tracker.json new file mode 100644 index 00000000..9486d775 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/online_resume__skills_development_tracker.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "online_resume__skills_development_tracker", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 349.7716019153595, + "token_usage": { + "input_tokens": 733127, + "output_tokens": 4388, + "total_tokens": 737515, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 2725.222920656204, + "token_usage": { + "input_tokens": 362322, + "output_tokens": 62964, + "total_tokens": 425286, + "reasoning_tokens": 56896 + }, + "turn_count": 20 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 549.9365866184235, + "token_usage": { + "input_tokens": 1559618, + "output_tokens": 5850, + "total_tokens": 1565468, + "reasoning_tokens": 0 + }, + "turn_count": 46 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 3084.928017139435, + "token_usage": { + "input_tokens": 233650, + "output_tokens": 63320, + "total_tokens": 296970, + "reasoning_tokens": 57188 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/online_resume__work_history_addition.json b/model_results/deepseek-v3-1-terminus-thinking/online_resume__work_history_addition.json new file mode 100644 index 00000000..c541fc6a --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/online_resume__work_history_addition.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "online_resume__work_history_addition", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 372.8326954841614, + "token_usage": { + "input_tokens": 551759, + "output_tokens": 5426, + "total_tokens": 557185, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1150.1524181365967, + "token_usage": { + "input_tokens": 154643, + "output_tokens": 27462, + "total_tokens": 182105, + "reasoning_tokens": 26398 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 2950 (char 2949)", + "execution_time": 201.71948194503784, + "token_usage": { + "input_tokens": 486528, + "output_tokens": 2340, + "total_tokens": 488868, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 268.417772769928, + "token_usage": { + "input_tokens": 6589, + "output_tokens": 6150, + "total_tokens": 12739, + "reasoning_tokens": 6082 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/papers__author_folders.json b/model_results/deepseek-v3-1-terminus-thinking/papers__author_folders.json new file mode 100644 index 00000000..f62fa26f --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/papers__author_folders.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "papers__author_folders", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092618312012020497545175016)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 516.0934507846832, + "token_usage": { + "input_tokens": 22534, + "output_tokens": 760, + "total_tokens": 23294, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 44.14991760253906, + "token_usage": { + "input_tokens": 3356, + "output_tokens": 947, + "total_tokens": 4303, + "reasoning_tokens": 914 + }, + "turn_count": 1 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\" (tid: 202509271315293757808694514516)\",\"type\":\"upstream_error\",\"param\":\"500\",\"code\":\"bad_response_status_code\"}}", + "execution_time": 1413.7718470096588, + "token_usage": { + "input_tokens": 33525, + "output_tokens": 14308, + "total_tokens": 47833, + "reasoning_tokens": 10709 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 419.44924664497375, + "token_usage": { + "input_tokens": 11335, + "output_tokens": 10591, + "total_tokens": 21926, + "reasoning_tokens": 10486 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/papers__find_math_paper.json b/model_results/deepseek-v3-1-terminus-thinking/papers__find_math_paper.json new file mode 100644 index 00000000..2b13446c --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/papers__find_math_paper.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "papers__find_math_paper", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092618382741013448290347244)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 375.5328314304352, + "token_usage": { + "input_tokens": 112242, + "output_tokens": 832, + "total_tokens": 113074, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\" (tid: 202509270231255736086653324120)\",\"type\":\"upstream_error\",\"param\":\"500\",\"code\":\"bad_response_status_code\"}}", + "execution_time": 1449.2769982814789, + "token_usage": { + "input_tokens": 10141, + "output_tokens": 11865, + "total_tokens": 22006, + "reasoning_tokens": 8399 + }, + "turn_count": 3 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\" (tid: 2025092802032851122197614006725)\",\"type\":\"upstream_error\",\"param\":\"500\",\"code\":\"bad_response_status_code\"}}", + "execution_time": 1257.1339843273163, + "token_usage": { + "input_tokens": 10141, + "output_tokens": 10952, + "total_tokens": 21093, + "reasoning_tokens": 7487 + }, + "turn_count": 3 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092723485586065064807028608)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1301.1785910129547, + "token_usage": { + "input_tokens": 10141, + "output_tokens": 14357, + "total_tokens": 24498, + "reasoning_tokens": 10890 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/papers__organize_legacy_papers.json b/model_results/deepseek-v3-1-terminus-thinking/papers__organize_legacy_papers.json new file mode 100644 index 00000000..998ef255 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/papers__organize_legacy_papers.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "papers__organize_legacy_papers", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 202509261846536869584268369530)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 540.0204980373383, + "token_usage": { + "input_tokens": 39292, + "output_tokens": 1057, + "total_tokens": 40349, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092702451335720203824260622)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 718.3258099555969, + "token_usage": { + "input_tokens": 11953, + "output_tokens": 4826, + "total_tokens": 16779, + "reasoning_tokens": 4256 + }, + "turn_count": 3 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092713581810491443809865989)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1577.7872943878174, + "token_usage": { + "input_tokens": 90002, + "output_tokens": 17456, + "total_tokens": 107458, + "reasoning_tokens": 14790 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092723572641629301728164063)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 315.5271635055542, + "token_usage": { + "input_tokens": 11952, + "output_tokens": 4239, + "total_tokens": 16191, + "reasoning_tokens": 3692 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/python_roadmap__expert_level_lessons.json b/model_results/deepseek-v3-1-terminus-thinking/python_roadmap__expert_level_lessons.json new file mode 100644 index 00000000..11144ab7 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/python_roadmap__expert_level_lessons.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "python_roadmap__expert_level_lessons", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 237.7851574420929, + "token_usage": { + "input_tokens": 591789, + "output_tokens": 2760, + "total_tokens": 594549, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 2144.2917325496674, + "token_usage": { + "input_tokens": 352022, + "output_tokens": 33904, + "total_tokens": 385926, + "reasoning_tokens": 31153 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 409 (char 408)", + "execution_time": 97.71788716316223, + "token_usage": { + "input_tokens": 154468, + "output_tokens": 879, + "total_tokens": 155347, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1801.4580142498016, + "token_usage": { + "input_tokens": 201954, + "output_tokens": 39304, + "total_tokens": 241258, + "reasoning_tokens": 36610 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/python_roadmap__learning_metrics_dashboard.json b/model_results/deepseek-v3-1-terminus-thinking/python_roadmap__learning_metrics_dashboard.json new file mode 100644 index 00000000..6df3d7f1 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/python_roadmap__learning_metrics_dashboard.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "python_roadmap__learning_metrics_dashboard", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 167.33575439453125, + "token_usage": { + "input_tokens": 355710, + "output_tokens": 2000, + "total_tokens": 357710, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1292.4404141902924, + "token_usage": { + "input_tokens": 71243, + "output_tokens": 32256, + "total_tokens": 103499, + "reasoning_tokens": 31861 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 155.6458375453949, + "token_usage": { + "input_tokens": 368623, + "output_tokens": 1947, + "total_tokens": 370570, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 2841.3382205963135, + "token_usage": { + "input_tokens": 341306, + "output_tokens": 66012, + "total_tokens": 407318, + "reasoning_tokens": 60344 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/reddit__ai_data_analyst.json b/model_results/deepseek-v3-1-terminus-thinking/reddit__ai_data_analyst.json new file mode 100644 index 00000000..c347a249 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/reddit__ai_data_analyst.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "reddit__ai_data_analyst", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 134310 tokens (134310 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 227.35113620758057, + "token_usage": { + "input_tokens": 706160, + "output_tokens": 1251, + "total_tokens": 707411, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092620512924986416699503630)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 265.55537009239197, + "token_usage": { + "input_tokens": 759976, + "output_tokens": 1161, + "total_tokens": 761137, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092704030360508659047430958)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1014.8283731937408, + "token_usage": { + "input_tokens": 468098, + "output_tokens": 14526, + "total_tokens": 482624, + "reasoning_tokens": 13823 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 202509271601262948474596930604)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1022.283301115036, + "token_usage": { + "input_tokens": 338388, + "output_tokens": 11658, + "total_tokens": 350046, + "reasoning_tokens": 10916 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/reddit__budget_europe_travel.json b/model_results/deepseek-v3-1-terminus-thinking/reddit__budget_europe_travel.json new file mode 100644 index 00000000..e80a37b8 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/reddit__budget_europe_travel.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "reddit__budget_europe_travel", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 318.70132970809937, + "token_usage": { + "input_tokens": 777622, + "output_tokens": 2497, + "total_tokens": 780119, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 264.9030439853668, + "token_usage": { + "input_tokens": 705056, + "output_tokens": 2418, + "total_tokens": 707474, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 1294.1408152580261, + "token_usage": { + "input_tokens": 693305, + "output_tokens": 19917, + "total_tokens": 713222, + "reasoning_tokens": 18071 + }, + "turn_count": 26 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092716274098540780961726280)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1498.355031967163, + "token_usage": { + "input_tokens": 724980, + "output_tokens": 17927, + "total_tokens": 742907, + "reasoning_tokens": 15935 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/reddit__buyitforlife_research.json b/model_results/deepseek-v3-1-terminus-thinking/reddit__buyitforlife_research.json new file mode 100644 index 00000000..641f4d68 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/reddit__buyitforlife_research.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "reddit__buyitforlife_research", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092615470127204972945798202)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 485.3865087032318, + "token_usage": { + "input_tokens": 233867, + "output_tokens": 800, + "total_tokens": 234667, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092620595376761897460684154)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 146.7022669315338, + "token_usage": { + "input_tokens": 234353, + "output_tokens": 855, + "total_tokens": 235208, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 187.0119686126709, + "token_usage": { + "input_tokens": 33241, + "output_tokens": 3757, + "total_tokens": 36998, + "reasoning_tokens": 3457 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092716395223596904978104796)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 696.3547940254211, + "token_usage": { + "input_tokens": 126105, + "output_tokens": 7768, + "total_tokens": 133873, + "reasoning_tokens": 7290 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/reddit__llm_research_summary.json b/model_results/deepseek-v3-1-terminus-thinking/reddit__llm_research_summary.json new file mode 100644 index 00000000..f53d8f4c --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/reddit__llm_research_summary.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "reddit__llm_research_summary", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092616005557128928348128585)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 806.8626036643982, + "token_usage": { + "input_tokens": 783458, + "output_tokens": 4273, + "total_tokens": 787731, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092621051522285895501322611)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 272.061527967453, + "token_usage": { + "input_tokens": 583775, + "output_tokens": 1834, + "total_tokens": 585609, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092705044681982690588463882)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 2122.9438638687134, + "token_usage": { + "input_tokens": 287862, + "output_tokens": 36631, + "total_tokens": 324493, + "reasoning_tokens": 30859 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1961.2656562328339, + "token_usage": { + "input_tokens": 485983, + "output_tokens": 47180, + "total_tokens": 533163, + "reasoning_tokens": 45435 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/reddit__movie_reviewer_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/reddit__movie_reviewer_analysis.json new file mode 100644 index 00000000..0dad1d48 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/reddit__movie_reviewer_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "reddit__movie_reviewer_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 202509261612343516419830068182)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 587.6891911029816, + "token_usage": { + "input_tokens": 675092, + "output_tokens": 1879, + "total_tokens": 676971, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092621103482432890946961965)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 276.3106234073639, + "token_usage": { + "input_tokens": 554078, + "output_tokens": 2416, + "total_tokens": 556494, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 3022.8521876335144, + "token_usage": { + "input_tokens": 587727, + "output_tokens": 72481, + "total_tokens": 660208, + "reasoning_tokens": 71015 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 16 (char 15)", + "execution_time": 1093.3647103309631, + "token_usage": { + "input_tokens": 177955, + "output_tokens": 28813, + "total_tokens": 206768, + "reasoning_tokens": 27934 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/reddit__nba_statistics_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/reddit__nba_statistics_analysis.json new file mode 100644 index 00000000..35d1ce1e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/reddit__nba_statistics_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "reddit__nba_statistics_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092616234683275736405540257)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 605.2492165565491, + "token_usage": { + "input_tokens": 934805, + "output_tokens": 2383, + "total_tokens": 937188, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092621150618752142711466807)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 224.2045509815216, + "token_usage": { + "input_tokens": 335957, + "output_tokens": 1736, + "total_tokens": 337693, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092706163954260042269484246)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1158.9364974498749, + "token_usage": { + "input_tokens": 275592, + "output_tokens": 17246, + "total_tokens": 292838, + "reasoning_tokens": 16681 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": "Extra data: line 1 column 244 (char 243)", + "execution_time": 1046.6779713630676, + "token_usage": { + "input_tokens": 240010, + "output_tokens": 26666, + "total_tokens": 266676, + "reasoning_tokens": 26054 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/reddit__routine_tracker_forum.json b/model_results/deepseek-v3-1-terminus-thinking/reddit__routine_tracker_forum.json new file mode 100644 index 00000000..e62f65e9 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/reddit__routine_tracker_forum.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "reddit__routine_tracker_forum", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092616330991954570570733292)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 541.5869607925415, + "token_usage": { + "input_tokens": 194865, + "output_tokens": 748, + "total_tokens": 195613, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 202509262118565076991602039656)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 181.92701196670532, + "token_usage": { + "input_tokens": 194847, + "output_tokens": 729, + "total_tokens": 195576, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092706290446272196759952488)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 704.2251386642456, + "token_usage": { + "input_tokens": 62899, + "output_tokens": 8453, + "total_tokens": 71352, + "reasoning_tokens": 7883 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092717590635031163055375234)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 473.9642276763916, + "token_usage": { + "input_tokens": 75383, + "output_tokens": 5653, + "total_tokens": 81036, + "reasoning_tokens": 5195 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/security__rls_business_access.json b/model_results/deepseek-v3-1-terminus-thinking/security__rls_business_access.json new file mode 100644 index 00000000..14f00c5b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/security__rls_business_access.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "security__rls_business_access", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 601.8553755283356, + "token_usage": { + "input_tokens": 553303, + "output_tokens": 9866, + "total_tokens": 563169, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 280.5147569179535, + "token_usage": { + "input_tokens": 230594, + "output_tokens": 4059, + "total_tokens": 234653, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1091.5518026351929, + "token_usage": { + "input_tokens": 59195, + "output_tokens": 31414, + "total_tokens": 90609, + "reasoning_tokens": 27365 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 736.2105123996735, + "token_usage": { + "input_tokens": 76180, + "output_tokens": 16699, + "total_tokens": 92879, + "reasoning_tokens": 10820 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/security__user_permission_audit.json b/model_results/deepseek-v3-1-terminus-thinking/security__user_permission_audit.json new file mode 100644 index 00000000..017d1f87 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/security__user_permission_audit.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "security__user_permission_audit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 310.8364751338959, + "token_usage": { + "input_tokens": 159466, + "output_tokens": 5560, + "total_tokens": 165026, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 314.9047932624817, + "token_usage": { + "input_tokens": 240764, + "output_tokens": 5295, + "total_tokens": 246059, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1416.4049904346466, + "token_usage": { + "input_tokens": 142863, + "output_tokens": 38649, + "total_tokens": 181512, + "reasoning_tokens": 35302 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 158.8768002986908, + "token_usage": { + "input_tokens": 7078, + "output_tokens": 4165, + "total_tokens": 11243, + "reasoning_tokens": 4129 + }, + "turn_count": 2 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/self_assessment__faq_column_layout.json b/model_results/deepseek-v3-1-terminus-thinking/self_assessment__faq_column_layout.json new file mode 100644 index 00000000..dc0800a8 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/self_assessment__faq_column_layout.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "self_assessment__faq_column_layout", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 193.1650745868683, + "token_usage": { + "input_tokens": 516625, + "output_tokens": 1811, + "total_tokens": 518436, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1562.039703130722, + "token_usage": { + "input_tokens": 151669, + "output_tokens": 37060, + "total_tokens": 188729, + "reasoning_tokens": 35734 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 191.101984500885, + "token_usage": { + "input_tokens": 438906, + "output_tokens": 1939, + "total_tokens": 440845, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 1171.1003983020782, + "token_usage": { + "input_tokens": 174715, + "output_tokens": 27374, + "total_tokens": 202089, + "reasoning_tokens": 24938 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/self_assessment__hyperfocus_analysis_report.json b/model_results/deepseek-v3-1-terminus-thinking/self_assessment__hyperfocus_analysis_report.json new file mode 100644 index 00000000..a56c5e00 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/self_assessment__hyperfocus_analysis_report.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "self_assessment__hyperfocus_analysis_report", + "runs": { + "run-1": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 17067 (char 17066)", + "execution_time": 594.8983533382416, + "token_usage": { + "input_tokens": 107073, + "output_tokens": 15837, + "total_tokens": 122910, + "reasoning_tokens": 11387 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1396.3053958415985, + "token_usage": { + "input_tokens": 132588, + "output_tokens": 34353, + "total_tokens": 166941, + "reasoning_tokens": 31282 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 299.3982877731323, + "token_usage": { + "input_tokens": 397273, + "output_tokens": 5127, + "total_tokens": 402400, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 296.9515995979309, + "token_usage": { + "input_tokens": 6806, + "output_tokens": 7675, + "total_tokens": 14481, + "reasoning_tokens": 7615 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/self_assessment__numbered_list_emojis.json b/model_results/deepseek-v3-1-terminus-thinking/self_assessment__numbered_list_emojis.json new file mode 100644 index 00000000..e8bb2f69 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/self_assessment__numbered_list_emojis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "self_assessment__numbered_list_emojis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 601.8822836875916, + "token_usage": { + "input_tokens": 106289, + "output_tokens": 16755, + "total_tokens": 123044, + "reasoning_tokens": 14120 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 842.8231256008148, + "token_usage": { + "input_tokens": 72386, + "output_tokens": 20149, + "total_tokens": 92535, + "reasoning_tokens": 18592 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 78.12506318092346, + "token_usage": { + "input_tokens": 171086, + "output_tokens": 771, + "total_tokens": 171857, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 793.7379803657532, + "token_usage": { + "input_tokens": 71480, + "output_tokens": 19866, + "total_tokens": 91346, + "reasoning_tokens": 18900 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping__advanced_product_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/shopping__advanced_product_analysis.json new file mode 100644 index 00000000..868a150e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping__advanced_product_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping__advanced_product_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092616445754852166594872864)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 629.6949858665466, + "token_usage": { + "input_tokens": 1040189, + "output_tokens": 1147, + "total_tokens": 1041336, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092621244078867692700332532)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 275.84868359565735, + "token_usage": { + "input_tokens": 947733, + "output_tokens": 1243, + "total_tokens": 948976, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092706540068478735696723810)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1413.9752473831177, + "token_usage": { + "input_tokens": 542616, + "output_tokens": 18572, + "total_tokens": 561188, + "reasoning_tokens": 17857 + }, + "turn_count": 13 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092718191451721798109783902)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1111.8663656711578, + "token_usage": { + "input_tokens": 571768, + "output_tokens": 16929, + "total_tokens": 588697, + "reasoning_tokens": 16228 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping__gaming_accessories_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/shopping__gaming_accessories_analysis.json new file mode 100644 index 00000000..13c05153 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping__gaming_accessories_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping__gaming_accessories_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092616574286724808942659427)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 686.551858663559, + "token_usage": { + "input_tokens": 1733890, + "output_tokens": 2749, + "total_tokens": 1736639, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092621545741973932995095114)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1752.1773190498352, + "token_usage": { + "input_tokens": 970153, + "output_tokens": 42333, + "total_tokens": 1012486, + "reasoning_tokens": 40926 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 45.972373723983765, + "token_usage": { + "input_tokens": 4366, + "output_tokens": 1074, + "total_tokens": 5440, + "reasoning_tokens": 1032 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 514.1069738864899, + "token_usage": { + "input_tokens": 105535, + "output_tokens": 13709, + "total_tokens": 119244, + "reasoning_tokens": 13490 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping__health_routine_optimization.json b/model_results/deepseek-v3-1-terminus-thinking/shopping__health_routine_optimization.json new file mode 100644 index 00000000..e3eabfde --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping__health_routine_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping__health_routine_optimization", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092617110138005193797822325)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 748.5469524860382, + "token_usage": { + "input_tokens": 1285621, + "output_tokens": 1653, + "total_tokens": 1287274, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092622132948452054192795419)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1048.2380719184875, + "token_usage": { + "input_tokens": 513984, + "output_tokens": 25598, + "total_tokens": 539582, + "reasoning_tokens": 24864 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092707350961586765554611737)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 2299.080590248108, + "token_usage": { + "input_tokens": 1002161, + "output_tokens": 33398, + "total_tokens": 1035559, + "reasoning_tokens": 32389 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092718543578731553705129134)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1483.6415514945984, + "token_usage": { + "input_tokens": 911088, + "output_tokens": 26239, + "total_tokens": 937327, + "reasoning_tokens": 25316 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping__holiday_baking_competition.json b/model_results/deepseek-v3-1-terminus-thinking/shopping__holiday_baking_competition.json new file mode 100644 index 00000000..29737251 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping__holiday_baking_competition.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping__holiday_baking_competition", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092617231386959448824468750)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 669.3894865512848, + "token_usage": { + "input_tokens": 2034502, + "output_tokens": 1938, + "total_tokens": 2036440, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092622252075394999964167704)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 646.8684825897217, + "token_usage": { + "input_tokens": 220907, + "output_tokens": 15861, + "total_tokens": 236768, + "reasoning_tokens": 14856 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092708091967403117135594956)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1921.1719779968262, + "token_usage": { + "input_tokens": 892104, + "output_tokens": 28581, + "total_tokens": 920685, + "reasoning_tokens": 27655 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092719200464434615842565579)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1431.1701622009277, + "token_usage": { + "input_tokens": 906297, + "output_tokens": 31203, + "total_tokens": 937500, + "reasoning_tokens": 30256 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping__multi_category_budget_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/shopping__multi_category_budget_analysis.json new file mode 100644 index 00000000..3e70d637 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping__multi_category_budget_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping__multi_category_budget_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092617331414448019517276453)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 518.6517353057861, + "token_usage": { + "input_tokens": 1350699, + "output_tokens": 1276, + "total_tokens": 1351975, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092622441810873624284272223)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1072.4663262367249, + "token_usage": { + "input_tokens": 725787, + "output_tokens": 24569, + "total_tokens": 750356, + "reasoning_tokens": 23899 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092708430274475306160420728)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 2045.283864736557, + "token_usage": { + "input_tokens": 922923, + "output_tokens": 28792, + "total_tokens": 951715, + "reasoning_tokens": 27915 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1203.1308624744415, + "token_usage": { + "input_tokens": 719822, + "output_tokens": 30385, + "total_tokens": 750207, + "reasoning_tokens": 28955 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping__printer_keyboard_search.json b/model_results/deepseek-v3-1-terminus-thinking/shopping__printer_keyboard_search.json new file mode 100644 index 00000000..ccb32792 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping__printer_keyboard_search.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping__printer_keyboard_search", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 251.4486665725708, + "token_usage": { + "input_tokens": 1028158, + "output_tokens": 1516, + "total_tokens": 1029674, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 806.9798963069916, + "token_usage": { + "input_tokens": 558587, + "output_tokens": 18650, + "total_tokens": 577237, + "reasoning_tokens": 18014 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 202509270909245472360962207760)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1499.2226073741913, + "token_usage": { + "input_tokens": 449916, + "output_tokens": 26368, + "total_tokens": 476284, + "reasoning_tokens": 25624 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": "Extra data: line 1 column 34 (char 33)", + "execution_time": 306.85438871383667, + "token_usage": { + "input_tokens": 94908, + "output_tokens": 7599, + "total_tokens": 102507, + "reasoning_tokens": 7356 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping__running_shoes_purchase.json b/model_results/deepseek-v3-1-terminus-thinking/shopping__running_shoes_purchase.json new file mode 100644 index 00000000..8f9fd97a --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping__running_shoes_purchase.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping__running_shoes_purchase", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 185.48341250419617, + "token_usage": { + "input_tokens": 595723, + "output_tokens": 1669, + "total_tokens": 597392, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092623354655359000577488917)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 2192.8332035541534, + "token_usage": { + "input_tokens": 876020, + "output_tokens": 51393, + "total_tokens": 927413, + "reasoning_tokens": 50579 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input token limit is 98304 (tid: 2025092709415128528340311128309)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 1892.1069226264954, + "token_usage": { + "input_tokens": 577998, + "output_tokens": 34111, + "total_tokens": 612109, + "reasoning_tokens": 33387 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1839.153243303299, + "token_usage": { + "input_tokens": 988076, + "output_tokens": 46598, + "total_tokens": 1034674, + "reasoning_tokens": 45722 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__customer_segmentation_setup.json b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__customer_segmentation_setup.json new file mode 100644 index 00000000..06cdae6b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__customer_segmentation_setup.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping_admin__customer_segmentation_setup", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 188.39300274848938, + "token_usage": { + "input_tokens": 512646, + "output_tokens": 1522, + "total_tokens": 514168, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 50.723018407821655, + "token_usage": { + "input_tokens": 4499, + "output_tokens": 1253, + "total_tokens": 5752, + "reasoning_tokens": 1209 + }, + "turn_count": 1 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 772.9591863155365, + "token_usage": { + "input_tokens": 366312, + "output_tokens": 14087, + "total_tokens": 380399, + "reasoning_tokens": 13250 + }, + "turn_count": 13 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 417.7184669971466, + "token_usage": { + "input_tokens": 191829, + "output_tokens": 10434, + "total_tokens": 202263, + "reasoning_tokens": 9824 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__fitness_promotion_strategy.json b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__fitness_promotion_strategy.json new file mode 100644 index 00000000..32200708 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__fitness_promotion_strategy.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping_admin__fitness_promotion_strategy", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 202509261754127141911208722564)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 386.5096573829651, + "token_usage": { + "input_tokens": 286018, + "output_tokens": 1287, + "total_tokens": 287305, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 53.1364381313324, + "token_usage": { + "input_tokens": 4693, + "output_tokens": 1418, + "total_tokens": 6111, + "reasoning_tokens": 1374 + }, + "turn_count": 1 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092710074718872636901527014)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 702.2816944122314, + "token_usage": { + "input_tokens": 89318, + "output_tokens": 9545, + "total_tokens": 98863, + "reasoning_tokens": 9116 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 2079.1209819316864, + "token_usage": { + "input_tokens": 562407, + "output_tokens": 57413, + "total_tokens": 619820, + "reasoning_tokens": 56292 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__marketing_customer_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__marketing_customer_analysis.json new file mode 100644 index 00000000..75bf6e3b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__marketing_customer_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping_admin__marketing_customer_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 284.3517520427704, + "token_usage": { + "input_tokens": 918065, + "output_tokens": 2388, + "total_tokens": 920453, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 43.57337212562561, + "token_usage": { + "input_tokens": 4918, + "output_tokens": 1189, + "total_tokens": 6107, + "reasoning_tokens": 1148 + }, + "turn_count": 1 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1876.474007844925, + "token_usage": { + "input_tokens": 615052, + "output_tokens": 39059, + "total_tokens": 654111, + "reasoning_tokens": 37768 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1377.8506574630737, + "token_usage": { + "input_tokens": 853288, + "output_tokens": 36326, + "total_tokens": 889614, + "reasoning_tokens": 34954 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__ny_expansion_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__ny_expansion_analysis.json new file mode 100644 index 00000000..962931d3 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__ny_expansion_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping_admin__ny_expansion_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092618081448210910049702202)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 462.2413377761841, + "token_usage": { + "input_tokens": 737854, + "output_tokens": 1150, + "total_tokens": 739004, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 807.3879680633545, + "token_usage": { + "input_tokens": 192102, + "output_tokens": 22091, + "total_tokens": 214193, + "reasoning_tokens": 21376 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1701.5678527355194, + "token_usage": { + "input_tokens": 424785, + "output_tokens": 37679, + "total_tokens": 462464, + "reasoning_tokens": 36752 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1100.2699790000916, + "token_usage": { + "input_tokens": 217291, + "output_tokens": 30839, + "total_tokens": 248130, + "reasoning_tokens": 30091 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__products_sales_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__products_sales_analysis.json new file mode 100644 index 00000000..d04a499d --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__products_sales_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping_admin__products_sales_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092618142624821867923294833)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 326.7992069721222, + "token_usage": { + "input_tokens": 64845, + "output_tokens": 367, + "total_tokens": 65212, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 202509270003264476850538428593)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 517.8571081161499, + "token_usage": { + "input_tokens": 69644, + "output_tokens": 10748, + "total_tokens": 80392, + "reasoning_tokens": 10340 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092722521637174004321653562)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 358.83156967163086, + "token_usage": { + "input_tokens": 53575, + "output_tokens": 9362, + "total_tokens": 62937, + "reasoning_tokens": 9011 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092721521445537525192780158)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 401.4616425037384, + "token_usage": { + "input_tokens": 69527, + "output_tokens": 10315, + "total_tokens": 79842, + "reasoning_tokens": 9942 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__sales_inventory_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__sales_inventory_analysis.json new file mode 100644 index 00000000..cada0982 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__sales_inventory_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping_admin__sales_inventory_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092618211234210252292520357)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 362.4113190174103, + "token_usage": { + "input_tokens": 126111, + "output_tokens": 933, + "total_tokens": 127044, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 2025092700141263499910241659396)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 640.8920018672943, + "token_usage": { + "input_tokens": 82988, + "output_tokens": 12424, + "total_tokens": 95412, + "reasoning_tokens": 12000 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"input characters limit is 393216 (tid: 202509271126048462695534657885)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_argument\"}}", + "execution_time": 826.518173456192, + "token_usage": { + "input_tokens": 99299, + "output_tokens": 10965, + "total_tokens": 110264, + "reasoning_tokens": 10503 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 16 (char 15)", + "execution_time": 418.84778237342834, + "token_usage": { + "input_tokens": 4690, + "output_tokens": 12154, + "total_tokens": 16844, + "reasoning_tokens": 8058 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__search_filtering_operations.json b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__search_filtering_operations.json new file mode 100644 index 00000000..d3d62032 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/shopping_admin__search_filtering_operations.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "shopping_admin__search_filtering_operations", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 542.2417359352112, + "token_usage": { + "input_tokens": 2053652, + "output_tokens": 6404, + "total_tokens": 2060056, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 737.8025443553925, + "token_usage": { + "input_tokens": 120984, + "output_tokens": 19588, + "total_tokens": 140572, + "reasoning_tokens": 19109 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 230.39245128631592, + "token_usage": { + "input_tokens": 40188, + "output_tokens": 4917, + "total_tokens": 45105, + "reasoning_tokens": 4661 + }, + "turn_count": 5 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 2353.357953310013, + "token_usage": { + "input_tokens": 706343, + "output_tokens": 64813, + "total_tokens": 771156, + "reasoning_tokens": 63258 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/sports__baseball_player_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/sports__baseball_player_analysis.json new file mode 100644 index 00000000..81f10d42 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/sports__baseball_player_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "sports__baseball_player_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 501.9244418144226, + "token_usage": { + "input_tokens": 928853, + "output_tokens": 7284, + "total_tokens": 936137, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 482.90266966819763, + "token_usage": { + "input_tokens": 986703, + "output_tokens": 6934, + "total_tokens": 993637, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 98.48183274269104, + "token_usage": { + "input_tokens": 34696, + "output_tokens": 2212, + "total_tokens": 36908, + "reasoning_tokens": 2019 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 61.051225900650024, + "token_usage": { + "input_tokens": 2759, + "output_tokens": 1633, + "total_tokens": 4392, + "reasoning_tokens": 1601 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/sports__participant_report_optimization.json b/model_results/deepseek-v3-1-terminus-thinking/sports__participant_report_optimization.json new file mode 100644 index 00000000..5fc4603e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/sports__participant_report_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "sports__participant_report_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 185.82887816429138, + "token_usage": { + "input_tokens": 192305, + "output_tokens": 2527, + "total_tokens": 194832, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 226.8004548549652, + "token_usage": { + "input_tokens": 284288, + "output_tokens": 3460, + "total_tokens": 287748, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 628.985714673996, + "token_usage": { + "input_tokens": 123472, + "output_tokens": 15916, + "total_tokens": 139388, + "reasoning_tokens": 13929 + }, + "turn_count": 12 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 438.4819691181183, + "token_usage": { + "input_tokens": 24074, + "output_tokens": 11072, + "total_tokens": 35146, + "reasoning_tokens": 9607 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/sports__team_roster_management.json b/model_results/deepseek-v3-1-terminus-thinking/sports__team_roster_management.json new file mode 100644 index 00000000..02fbde5f --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/sports__team_roster_management.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "sports__team_roster_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 385.0476920604706, + "token_usage": { + "input_tokens": 919773, + "output_tokens": 4478, + "total_tokens": 924251, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 356.95028591156006, + "token_usage": { + "input_tokens": 605303, + "output_tokens": 4568, + "total_tokens": 609871, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + "run-3": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 11 (char 10)", + "execution_time": 677.6811947822571, + "token_usage": { + "input_tokens": 111057, + "output_tokens": 18782, + "total_tokens": 129839, + "reasoning_tokens": 17590 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1398.0381712913513, + "token_usage": { + "input_tokens": 126813, + "output_tokens": 32319, + "total_tokens": 159132, + "reasoning_tokens": 28703 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/standard_operating_procedure__deployment_process_sop.json b/model_results/deepseek-v3-1-terminus-thinking/standard_operating_procedure__deployment_process_sop.json new file mode 100644 index 00000000..56fee6f8 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/standard_operating_procedure__deployment_process_sop.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "standard_operating_procedure__deployment_process_sop", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 1545.4248478412628, + "token_usage": { + "input_tokens": 230617, + "output_tokens": 41104, + "total_tokens": 271721, + "reasoning_tokens": 35917 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": "Execution timed out after 3600 seconds", + "execution_time": 3600.03568315506, + "token_usage": { + "input_tokens": 817385, + "output_tokens": 61242, + "total_tokens": 878627, + "reasoning_tokens": 55535 + }, + "turn_count": 22 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 518.9985489845276, + "token_usage": { + "input_tokens": 2779599, + "output_tokens": 4445, + "total_tokens": 2784044, + "reasoning_tokens": 0 + }, + "turn_count": 49 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 1990.661957025528, + "token_usage": { + "input_tokens": 365481, + "output_tokens": 45234, + "total_tokens": 410715, + "reasoning_tokens": 37998 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/standard_operating_procedure__section_organization.json b/model_results/deepseek-v3-1-terminus-thinking/standard_operating_procedure__section_organization.json new file mode 100644 index 00000000..7dfc9a3e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/standard_operating_procedure__section_organization.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "standard_operating_procedure__section_organization", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 1947.6300837993622, + "token_usage": { + "input_tokens": 141731, + "output_tokens": 53367, + "total_tokens": 195098, + "reasoning_tokens": 51145 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 920.9546501636505, + "token_usage": { + "input_tokens": 30058, + "output_tokens": 22077, + "total_tokens": 52135, + "reasoning_tokens": 21702 + }, + "turn_count": 3 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 465.8426823616028, + "token_usage": { + "input_tokens": 947240, + "output_tokens": 6178, + "total_tokens": 953418, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 2128.761714696884, + "token_usage": { + "input_tokens": 79161, + "output_tokens": 35572, + "total_tokens": 114733, + "reasoning_tokens": 33225 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/student_database__duplicate_name.json b/model_results/deepseek-v3-1-terminus-thinking/student_database__duplicate_name.json new file mode 100644 index 00000000..b945b627 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/student_database__duplicate_name.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "student_database__duplicate_name", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 282.1767854690552, + "token_usage": { + "input_tokens": 199314, + "output_tokens": 5275, + "total_tokens": 204589, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 681.841598033905, + "token_usage": { + "input_tokens": 16785, + "output_tokens": 15730, + "total_tokens": 32515, + "reasoning_tokens": 15183 + }, + "turn_count": 4 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 615.7003190517426, + "token_usage": { + "input_tokens": 16810, + "output_tokens": 14991, + "total_tokens": 31801, + "reasoning_tokens": 14413 + }, + "turn_count": 4 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 636.8990454673767, + "token_usage": { + "input_tokens": 51838, + "output_tokens": 17767, + "total_tokens": 69605, + "reasoning_tokens": 17108 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/student_database__english_talent.json b/model_results/deepseek-v3-1-terminus-thinking/student_database__english_talent.json new file mode 100644 index 00000000..5d0d6414 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/student_database__english_talent.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "student_database__english_talent", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 853.4310433864594, + "token_usage": { + "input_tokens": 684997, + "output_tokens": 17670, + "total_tokens": 702667, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 12579 (char 12578)", + "execution_time": 745.1283392906189, + "token_usage": { + "input_tokens": 22620, + "output_tokens": 17823, + "total_tokens": 40443, + "reasoning_tokens": 13421 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 12623 (char 12622)", + "execution_time": 1314.1261701583862, + "token_usage": { + "input_tokens": 28325, + "output_tokens": 17500, + "total_tokens": 45825, + "reasoning_tokens": 13053 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 12417 (char 12416)", + "execution_time": 695.2106280326843, + "token_usage": { + "input_tokens": 82296, + "output_tokens": 19845, + "total_tokens": 102141, + "reasoning_tokens": 15304 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/student_database__gradebased_score.json b/model_results/deepseek-v3-1-terminus-thinking/student_database__gradebased_score.json new file mode 100644 index 00000000..6f0bf656 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/student_database__gradebased_score.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "student_database__gradebased_score", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 614.1367244720459, + "token_usage": { + "input_tokens": 792495, + "output_tokens": 12021, + "total_tokens": 804516, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1449.01841878891, + "token_usage": { + "input_tokens": 114831, + "output_tokens": 34643, + "total_tokens": 149474, + "reasoning_tokens": 23623 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 12593 (char 12592)", + "execution_time": 349.3603434562683, + "token_usage": { + "input_tokens": 23428, + "output_tokens": 8256, + "total_tokens": 31684, + "reasoning_tokens": 3983 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 12597 (char 12596)", + "execution_time": 327.54914593696594, + "token_usage": { + "input_tokens": 35565, + "output_tokens": 9291, + "total_tokens": 44856, + "reasoning_tokens": 4939 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/team_projects__priority_tasks_table.json b/model_results/deepseek-v3-1-terminus-thinking/team_projects__priority_tasks_table.json new file mode 100644 index 00000000..f084ff03 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/team_projects__priority_tasks_table.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "team_projects__priority_tasks_table", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 1642.3761992454529, + "token_usage": { + "input_tokens": 82585, + "output_tokens": 45525, + "total_tokens": 128110, + "reasoning_tokens": 44169 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 2037.2663569450378, + "token_usage": { + "input_tokens": 168450, + "output_tokens": 46370, + "total_tokens": 214820, + "reasoning_tokens": 42965 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 116.3914647102356, + "token_usage": { + "input_tokens": 118896, + "output_tokens": 1678, + "total_tokens": 120574, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1338.0457293987274, + "token_usage": { + "input_tokens": 72536, + "output_tokens": 29352, + "total_tokens": 101888, + "reasoning_tokens": 28150 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/team_projects__swap_tasks.json b/model_results/deepseek-v3-1-terminus-thinking/team_projects__swap_tasks.json new file mode 100644 index 00000000..432d312e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/team_projects__swap_tasks.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "team_projects__swap_tasks", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 711.5996866226196, + "token_usage": { + "input_tokens": 104798, + "output_tokens": 18779, + "total_tokens": 123577, + "reasoning_tokens": 17374 + }, + "turn_count": 5 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 1616.9762933254242, + "token_usage": { + "input_tokens": 207004, + "output_tokens": 40225, + "total_tokens": 247229, + "reasoning_tokens": 34600 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 359.2617642879486, + "token_usage": { + "input_tokens": 1532728, + "output_tokens": 3552, + "total_tokens": 1536280, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 1091.5245559215546, + "token_usage": { + "input_tokens": 140480, + "output_tokens": 25509, + "total_tokens": 165989, + "reasoning_tokens": 23757 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/threestudio__code_locating.json b/model_results/deepseek-v3-1-terminus-thinking/threestudio__code_locating.json new file mode 100644 index 00000000..64e8f779 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/threestudio__code_locating.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "threestudio__code_locating", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 89.58234930038452, + "token_usage": { + "input_tokens": 57810, + "output_tokens": 1025, + "total_tokens": 58835, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 399.82364892959595, + "token_usage": { + "input_tokens": 57073, + "output_tokens": 8754, + "total_tokens": 65827, + "reasoning_tokens": 7933 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 39.92405438423157, + "token_usage": { + "input_tokens": 3075, + "output_tokens": 970, + "total_tokens": 4045, + "reasoning_tokens": 936 + }, + "turn_count": 1 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 197.1846022605896, + "token_usage": { + "input_tokens": 38632, + "output_tokens": 4866, + "total_tokens": 43498, + "reasoning_tokens": 4450 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/threestudio__output_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/threestudio__output_analysis.json new file mode 100644 index 00000000..d32a2e71 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/threestudio__output_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "threestudio__output_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 175.36647152900696, + "token_usage": { + "input_tokens": 320700, + "output_tokens": 2055, + "total_tokens": 322755, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 466.0471348762512, + "token_usage": { + "input_tokens": 72578, + "output_tokens": 10920, + "total_tokens": 83498, + "reasoning_tokens": 10055 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 512.9723234176636, + "token_usage": { + "input_tokens": 41169, + "output_tokens": 11496, + "total_tokens": 52665, + "reasoning_tokens": 11030 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 538.5606427192688, + "token_usage": { + "input_tokens": 60797, + "output_tokens": 14582, + "total_tokens": 75379, + "reasoning_tokens": 13847 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/threestudio__requirements_completion.json b/model_results/deepseek-v3-1-terminus-thinking/threestudio__requirements_completion.json new file mode 100644 index 00000000..0cc57290 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/threestudio__requirements_completion.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "threestudio__requirements_completion", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 99.41771006584167, + "token_usage": { + "input_tokens": 146044, + "output_tokens": 1057, + "total_tokens": 147101, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1047.792384147644, + "token_usage": { + "input_tokens": 158949, + "output_tokens": 24397, + "total_tokens": 183346, + "reasoning_tokens": 23146 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 763.8339810371399, + "token_usage": { + "input_tokens": 101466, + "output_tokens": 16985, + "total_tokens": 118451, + "reasoning_tokens": 16034 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 236.0360825061798, + "token_usage": { + "input_tokens": 32746, + "output_tokens": 5710, + "total_tokens": 38456, + "reasoning_tokens": 5311 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/toronto_guide__change_color.json b/model_results/deepseek-v3-1-terminus-thinking/toronto_guide__change_color.json new file mode 100644 index 00000000..0ecbd8e1 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/toronto_guide__change_color.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "toronto_guide__change_color", + "runs": { + "run-1": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 2276 (char 2275)", + "execution_time": 408.6908829212189, + "token_usage": { + "input_tokens": 43453, + "output_tokens": 11372, + "total_tokens": 54825, + "reasoning_tokens": 10006 + }, + "turn_count": 4 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1304.6740472316742, + "token_usage": { + "input_tokens": 222048, + "output_tokens": 32273, + "total_tokens": 254321, + "reasoning_tokens": 27184 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 227.67290902137756, + "token_usage": { + "input_tokens": 318265, + "output_tokens": 3079, + "total_tokens": 321344, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1064.318662405014, + "token_usage": { + "input_tokens": 115244, + "output_tokens": 24030, + "total_tokens": 139274, + "reasoning_tokens": 20416 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/toronto_guide__weekend_adventure_planner.json b/model_results/deepseek-v3-1-terminus-thinking/toronto_guide__weekend_adventure_planner.json new file mode 100644 index 00000000..03925c51 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/toronto_guide__weekend_adventure_planner.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "notion", + "task": "toronto_guide__weekend_adventure_planner", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 589.1192388534546, + "token_usage": { + "input_tokens": 62560, + "output_tokens": 17328, + "total_tokens": 79888, + "reasoning_tokens": 15194 + }, + "turn_count": 4 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 707.2529714107513, + "token_usage": { + "input_tokens": 62524, + "output_tokens": 18956, + "total_tokens": 81480, + "reasoning_tokens": 16760 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 137.0676281452179, + "token_usage": { + "input_tokens": 179979, + "output_tokens": 1842, + "total_tokens": 181821, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 1131.1666383743286, + "token_usage": { + "input_tokens": 85989, + "output_tokens": 25167, + "total_tokens": 111156, + "reasoning_tokens": 21616 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/vectors__dba_vector_analysis.json b/model_results/deepseek-v3-1-terminus-thinking/vectors__dba_vector_analysis.json new file mode 100644 index 00000000..4798e239 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/vectors__dba_vector_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "postgres", + "task": "vectors__dba_vector_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 474.7970817089081, + "token_usage": { + "input_tokens": 500603, + "output_tokens": 6968, + "total_tokens": 507571, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 607.1480605602264, + "token_usage": { + "input_tokens": 611079, + "output_tokens": 10136, + "total_tokens": 621215, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 381.4060699939728, + "token_usage": { + "input_tokens": 16683, + "output_tokens": 9772, + "total_tokens": 26455, + "reasoning_tokens": 8464 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": "Extra data: line 1 column 238 (char 237)", + "execution_time": 766.287832736969, + "token_usage": { + "input_tokens": 9219, + "output_tokens": 13092, + "total_tokens": 22311, + "reasoning_tokens": 12792 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/votenet__dataset_comparison.json b/model_results/deepseek-v3-1-terminus-thinking/votenet__dataset_comparison.json new file mode 100644 index 00000000..1e22ab1b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/votenet__dataset_comparison.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "votenet__dataset_comparison", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 169.3777139186859, + "token_usage": { + "input_tokens": 332261, + "output_tokens": 1554, + "total_tokens": 333815, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 567.3579223155975, + "token_usage": { + "input_tokens": 91057, + "output_tokens": 12855, + "total_tokens": 103912, + "reasoning_tokens": 12189 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1373.9668321609497, + "token_usage": { + "input_tokens": 206322, + "output_tokens": 33289, + "total_tokens": 239611, + "reasoning_tokens": 32258 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 149.29404830932617, + "token_usage": { + "input_tokens": 33057, + "output_tokens": 3498, + "total_tokens": 36555, + "reasoning_tokens": 3105 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/votenet__debugging.json b/model_results/deepseek-v3-1-terminus-thinking/votenet__debugging.json new file mode 100644 index 00000000..8e4aba54 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/votenet__debugging.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "votenet__debugging", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 484.8111913204193, + "token_usage": { + "input_tokens": 640345, + "output_tokens": 8118, + "total_tokens": 648463, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 400.0892035961151, + "token_usage": { + "input_tokens": 56832, + "output_tokens": 8760, + "total_tokens": 65592, + "reasoning_tokens": 8004 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 213.07048678398132, + "token_usage": { + "input_tokens": 34136, + "output_tokens": 5256, + "total_tokens": 39392, + "reasoning_tokens": 4879 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 562.9290969371796, + "token_usage": { + "input_tokens": 56680, + "output_tokens": 14530, + "total_tokens": 71210, + "reasoning_tokens": 13771 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/votenet__requirements_writing.json b/model_results/deepseek-v3-1-terminus-thinking/votenet__requirements_writing.json new file mode 100644 index 00000000..3047ce43 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/votenet__requirements_writing.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "filesystem", + "task": "votenet__requirements_writing", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 174.5799949169159, + "token_usage": { + "input_tokens": 160787, + "output_tokens": 1960, + "total_tokens": 162747, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 625.5320510864258, + "token_usage": { + "input_tokens": 172927, + "output_tokens": 12635, + "total_tokens": 185562, + "reasoning_tokens": 11729 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 138.60280013084412, + "token_usage": { + "input_tokens": 29916, + "output_tokens": 3131, + "total_tokens": 33047, + "reasoning_tokens": 2739 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 418.7612109184265, + "token_usage": { + "input_tokens": 88672, + "output_tokens": 9965, + "total_tokens": 98637, + "reasoning_tokens": 9062 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/web_search__birth_of_arvinxu.json b/model_results/deepseek-v3-1-terminus-thinking/web_search__birth_of_arvinxu.json new file mode 100644 index 00000000..aad6cf33 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/web_search__birth_of_arvinxu.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "web_search__birth_of_arvinxu", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 71.80601024627686, + "token_usage": { + "input_tokens": 119963, + "output_tokens": 722, + "total_tokens": 120685, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 160.10860681533813, + "token_usage": { + "input_tokens": 544679, + "output_tokens": 1267, + "total_tokens": 545946, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 140.54830026626587, + "token_usage": { + "input_tokens": 204727, + "output_tokens": 1315, + "total_tokens": 206042, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 142.61875700950623, + "token_usage": { + "input_tokens": 415635, + "output_tokens": 1226, + "total_tokens": 416861, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus-thinking/web_search__r1_arxiv.json b/model_results/deepseek-v3-1-terminus-thinking/web_search__r1_arxiv.json new file mode 100644 index 00000000..987a7a42 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus-thinking/web_search__r1_arxiv.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus-thinking", + "service": "playwright", + "task": "web_search__r1_arxiv", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 147621 tokens (147621 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 125.5896155834198, + "token_usage": { + "input_tokens": 335147, + "output_tokens": 557, + "total_tokens": 335704, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 144184 tokens (144184 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 144.76980304718018, + "token_usage": { + "input_tokens": 408530, + "output_tokens": 616, + "total_tokens": 409146, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 160714 tokens (160714 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 133.10206604003906, + "token_usage": { + "input_tokens": 284352, + "output_tokens": 506, + "total_tokens": 284858, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 135756 tokens (135756 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 201.1830825805664, + "token_usage": { + "input_tokens": 985933, + "output_tokens": 859, + "total_tokens": 986792, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/build_your_own_x__find_commit_date.json b/model_results/deepseek-v3-1-terminus/build_your_own_x__find_commit_date.json new file mode 100644 index 00000000..35fc9abb --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/build_your_own_x__find_commit_date.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "build_your_own_x__find_commit_date", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 388.3499140739441, + "token_usage": { + "input_tokens": 1435464, + "output_tokens": 2579, + "total_tokens": 1438043, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092621500017558617784443320)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 12.15988540649414, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 12.474773406982422, + "token_usage": { + "input_tokens": 23533, + "output_tokens": 95, + "total_tokens": 23628, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 518.7657308578491, + "token_usage": { + "input_tokens": 1087397, + "output_tokens": 1637, + "total_tokens": 1089034, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/build_your_own_x__find_rag_commit.json b/model_results/deepseek-v3-1-terminus/build_your_own_x__find_rag_commit.json new file mode 100644 index 00000000..e843ca62 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/build_your_own_x__find_rag_commit.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "build_your_own_x__find_rag_commit", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 208.98038411140442, + "token_usage": { + "input_tokens": 537624, + "output_tokens": 1453, + "total_tokens": 539077, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092621510878828353842014541)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 9.882914543151855, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 484.1384336948395, + "token_usage": { + "input_tokens": 856091, + "output_tokens": 1864, + "total_tokens": 857955, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 73 (char 72)", + "execution_time": 732.2848217487335, + "token_usage": { + "input_tokens": 1479523, + "output_tokens": 1166, + "total_tokens": 1480689, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/chinook__customer_data_migration.json b/model_results/deepseek-v3-1-terminus/chinook__customer_data_migration.json new file mode 100644 index 00000000..64472d69 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/chinook__customer_data_migration.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "chinook__customer_data_migration", + "runs": { + "run-1": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 9 (char 8)", + "execution_time": 203.7839663028717, + "token_usage": { + "input_tokens": 61296, + "output_tokens": 4233, + "total_tokens": 65529, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 9 (char 8)", + "execution_time": 222.6151738166809, + "token_usage": { + "input_tokens": 74897, + "output_tokens": 4282, + "total_tokens": 79179, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 9 (char 8)", + "execution_time": 205.81352043151855, + "token_usage": { + "input_tokens": 74958, + "output_tokens": 4297, + "total_tokens": 79255, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623574170149976377570286)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.747117757797241, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/chinook__employee_hierarchy_management.json b/model_results/deepseek-v3-1-terminus/chinook__employee_hierarchy_management.json new file mode 100644 index 00000000..77796044 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/chinook__employee_hierarchy_management.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "chinook__employee_hierarchy_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 181.4918601512909, + "token_usage": { + "input_tokens": 185888, + "output_tokens": 2413, + "total_tokens": 188301, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 201.84227919578552, + "token_usage": { + "input_tokens": 249784, + "output_tokens": 2340, + "total_tokens": 252124, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 200.8980212211609, + "token_usage": { + "input_tokens": 261562, + "output_tokens": 2313, + "total_tokens": 263875, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623575025640681556923744)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.158531904220581, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/chinook__sales_and_music_charts.json b/model_results/deepseek-v3-1-terminus/chinook__sales_and_music_charts.json new file mode 100644 index 00000000..468b4a62 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/chinook__sales_and_music_charts.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "chinook__sales_and_music_charts", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 270.4434940814972, + "token_usage": { + "input_tokens": 340352, + "output_tokens": 4035, + "total_tokens": 344387, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 255.62756371498108, + "token_usage": { + "input_tokens": 263974, + "output_tokens": 3607, + "total_tokens": 267581, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 290.57821702957153, + "token_usage": { + "input_tokens": 390092, + "output_tokens": 3995, + "total_tokens": 394087, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623575922934618745192893)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.829689741134644, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/claude-code__automated_changelog_generation.json b/model_results/deepseek-v3-1-terminus/claude-code__automated_changelog_generation.json new file mode 100644 index 00000000..e13c5178 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/claude-code__automated_changelog_generation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "claude-code__automated_changelog_generation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 473.3854594230652, + "token_usage": { + "input_tokens": 827315, + "output_tokens": 7174, + "total_tokens": 834489, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092621531555556655514640416)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.389399766921997, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 443.25855326652527, + "token_usage": { + "input_tokens": 537077, + "output_tokens": 5536, + "total_tokens": 542613, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 4157 (char 4156)", + "execution_time": 169.28932094573975, + "token_usage": { + "input_tokens": 153052, + "output_tokens": 2790, + "total_tokens": 155842, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/claude-code__claude_collaboration_analysis.json b/model_results/deepseek-v3-1-terminus/claude-code__claude_collaboration_analysis.json new file mode 100644 index 00000000..27e05309 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/claude-code__claude_collaboration_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "claude-code__claude_collaboration_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 111.53016495704651, + "token_usage": { + "input_tokens": 260912, + "output_tokens": 1286, + "total_tokens": 262198, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092621553534745950774611847)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 11.998449563980103, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 159.6641800403595, + "token_usage": { + "input_tokens": 287885, + "output_tokens": 791, + "total_tokens": 288676, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 66.93278241157532, + "token_usage": { + "input_tokens": 135269, + "output_tokens": 714, + "total_tokens": 135983, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/claude-code__critical_issue_hotfix_workflow.json b/model_results/deepseek-v3-1-terminus/claude-code__critical_issue_hotfix_workflow.json new file mode 100644 index 00000000..d65ce857 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/claude-code__critical_issue_hotfix_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "claude-code__critical_issue_hotfix_workflow", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 189.12998604774475, + "token_usage": { + "input_tokens": 220970, + "output_tokens": 2647, + "total_tokens": 223617, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092621575953163155515778758)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.897485256195068, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": "Extra data: line 1 column 86 (char 85)", + "execution_time": 278.516259431839, + "token_usage": { + "input_tokens": 300591, + "output_tokens": 2969, + "total_tokens": 303560, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 17.21670365333557, + "token_usage": { + "input_tokens": 24431, + "output_tokens": 276, + "total_tokens": 24707, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/claude-code__feature_commit_tracking.json b/model_results/deepseek-v3-1-terminus/claude-code__feature_commit_tracking.json new file mode 100644 index 00000000..ce3ce260 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/claude-code__feature_commit_tracking.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "claude-code__feature_commit_tracking", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 326.2840278148651, + "token_usage": { + "input_tokens": 1671658, + "output_tokens": 2548, + "total_tokens": 1674206, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622002122267481091684876)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.507361650466919, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 991.8688023090363, + "token_usage": { + "input_tokens": 1484538, + "output_tokens": 1838, + "total_tokens": 1486376, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 805.1356992721558, + "token_usage": { + "input_tokens": 1697275, + "output_tokens": 2297, + "total_tokens": 1699572, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/claude-code__label_color_standardization.json b/model_results/deepseek-v3-1-terminus/claude-code__label_color_standardization.json new file mode 100644 index 00000000..862da85e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/claude-code__label_color_standardization.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "claude-code__label_color_standardization", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 161142 tokens (161142 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092617005488972709673465100)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 70.51958465576172, + "token_usage": { + "input_tokens": 148346, + "output_tokens": 253, + "total_tokens": 148599, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 202509262202385621192580147892)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.07112979888916, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 103.45589923858643, + "token_usage": { + "input_tokens": 24064, + "output_tokens": 2264, + "total_tokens": 26328, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 316.0364558696747, + "token_usage": { + "input_tokens": 575619, + "output_tokens": 2715, + "total_tokens": 578334, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/company_in_a_box__employee_onboarding.json b/model_results/deepseek-v3-1-terminus/company_in_a_box__employee_onboarding.json new file mode 100644 index 00000000..d6e89ba1 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/company_in_a_box__employee_onboarding.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "company_in_a_box__employee_onboarding", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 419.08600211143494, + "token_usage": { + "input_tokens": 1559484, + "output_tokens": 4581, + "total_tokens": 1564065, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": true, + "error_message": "Expecting ',' delimiter: line 1 column 7833 (char 7832)", + "execution_time": 220.81275343894958, + "token_usage": { + "input_tokens": 284282, + "output_tokens": 3970, + "total_tokens": 288252, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": "Extra data: line 1 column 256 (char 255)", + "execution_time": 100.90077924728394, + "token_usage": { + "input_tokens": 198295, + "output_tokens": 705, + "total_tokens": 199000, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 346.02288794517517, + "token_usage": { + "input_tokens": 670581, + "output_tokens": 4181, + "total_tokens": 674762, + "reasoning_tokens": 0 + }, + "turn_count": 28 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/company_in_a_box__goals_restructure.json b/model_results/deepseek-v3-1-terminus/company_in_a_box__goals_restructure.json new file mode 100644 index 00000000..a3bad185 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/company_in_a_box__goals_restructure.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "company_in_a_box__goals_restructure", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 255.54564332962036, + "token_usage": { + "input_tokens": 526049, + "output_tokens": 2235, + "total_tokens": 528284, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 250.75043869018555, + "token_usage": { + "input_tokens": 405411, + "output_tokens": 2450, + "total_tokens": 407861, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 387.35196185112, + "token_usage": { + "input_tokens": 2260266, + "output_tokens": 2528, + "total_tokens": 2262794, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 324.52826261520386, + "token_usage": { + "input_tokens": 1252146, + "output_tokens": 2975, + "total_tokens": 1255121, + "reasoning_tokens": 0 + }, + "turn_count": 33 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/company_in_a_box__quarterly_review_dashboard.json b/model_results/deepseek-v3-1-terminus/company_in_a_box__quarterly_review_dashboard.json new file mode 100644 index 00000000..00605562 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/company_in_a_box__quarterly_review_dashboard.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "company_in_a_box__quarterly_review_dashboard", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 444.87319684028625, + "token_usage": { + "input_tokens": 1295955, + "output_tokens": 4674, + "total_tokens": 1300629, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 9.161523342132568, + "token_usage": { + "input_tokens": 6858, + "output_tokens": 106, + "total_tokens": 6964, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 720.7286939620972, + "token_usage": { + "input_tokens": 2342881, + "output_tokens": 8105, + "total_tokens": 2350986, + "reasoning_tokens": 0 + }, + "turn_count": 52 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 377.1310796737671, + "token_usage": { + "input_tokens": 1046479, + "output_tokens": 4124, + "total_tokens": 1050603, + "reasoning_tokens": 0 + }, + "turn_count": 31 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/computer_science_student_dashboard__code_snippets_go.json b/model_results/deepseek-v3-1-terminus/computer_science_student_dashboard__code_snippets_go.json new file mode 100644 index 00000000..c6f68a25 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/computer_science_student_dashboard__code_snippets_go.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "computer_science_student_dashboard__code_snippets_go", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 207.80923223495483, + "token_usage": { + "input_tokens": 321688, + "output_tokens": 2068, + "total_tokens": 323756, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 7.575263500213623, + "token_usage": { + "input_tokens": 6712, + "output_tokens": 99, + "total_tokens": 6811, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 517.6202044487, + "token_usage": { + "input_tokens": 867599, + "output_tokens": 6688, + "total_tokens": 874287, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 205.75380516052246, + "token_usage": { + "input_tokens": 310797, + "output_tokens": 2372, + "total_tokens": 313169, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/computer_science_student_dashboard__courses_internships_relation.json b/model_results/deepseek-v3-1-terminus/computer_science_student_dashboard__courses_internships_relation.json new file mode 100644 index 00000000..280d0cfe --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/computer_science_student_dashboard__courses_internships_relation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "computer_science_student_dashboard__courses_internships_relation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 319.4499981403351, + "token_usage": { + "input_tokens": 633921, + "output_tokens": 3584, + "total_tokens": 637505, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 7.643780946731567, + "token_usage": { + "input_tokens": 6886, + "output_tokens": 102, + "total_tokens": 6988, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 328.9060800075531, + "token_usage": { + "input_tokens": 447623, + "output_tokens": 3449, + "total_tokens": 451072, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-4": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 281 (char 280)", + "execution_time": 32.541977405548096, + "token_usage": { + "input_tokens": 26376, + "output_tokens": 307, + "total_tokens": 26683, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/computer_science_student_dashboard__study_session_tracker.json b/model_results/deepseek-v3-1-terminus/computer_science_student_dashboard__study_session_tracker.json new file mode 100644 index 00000000..f447da01 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/computer_science_student_dashboard__study_session_tracker.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "computer_science_student_dashboard__study_session_tracker", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 170.7622299194336, + "token_usage": { + "input_tokens": 125193, + "output_tokens": 1188, + "total_tokens": 126381, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 215.09980010986328, + "token_usage": { + "input_tokens": 431310, + "output_tokens": 1208, + "total_tokens": 432518, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 108.98834300041199, + "token_usage": { + "input_tokens": 124620, + "output_tokens": 1147, + "total_tokens": 125767, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 112.19266366958618, + "token_usage": { + "input_tokens": 143512, + "output_tokens": 1269, + "total_tokens": 144781, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/desktop__music_report.json b/model_results/deepseek-v3-1-terminus/desktop__music_report.json new file mode 100644 index 00000000..dfccc913 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/desktop__music_report.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "desktop__music_report", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 129.30624628067017, + "token_usage": { + "input_tokens": 52687, + "output_tokens": 1769, + "total_tokens": 54456, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 118.57634544372559, + "token_usage": { + "input_tokens": 47130, + "output_tokens": 1709, + "total_tokens": 48839, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 79.30341839790344, + "token_usage": { + "input_tokens": 36390, + "output_tokens": 1619, + "total_tokens": 38009, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 52.89259696006775, + "token_usage": { + "input_tokens": 28004, + "output_tokens": 855, + "total_tokens": 28859, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/desktop__project_management.json b/model_results/deepseek-v3-1-terminus/desktop__project_management.json new file mode 100644 index 00000000..4235aa1a --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/desktop__project_management.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "desktop__project_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 391.0343129634857, + "token_usage": { + "input_tokens": 310079, + "output_tokens": 4737, + "total_tokens": 314816, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 353.55652832984924, + "token_usage": { + "input_tokens": 284959, + "output_tokens": 4228, + "total_tokens": 289187, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 307.50742197036743, + "token_usage": { + "input_tokens": 118280, + "output_tokens": 6289, + "total_tokens": 124569, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 205.3735911846161, + "token_usage": { + "input_tokens": 90657, + "output_tokens": 4949, + "total_tokens": 95606, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/desktop__timeline_extraction.json b/model_results/deepseek-v3-1-terminus/desktop__timeline_extraction.json new file mode 100644 index 00000000..d3594767 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/desktop__timeline_extraction.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "desktop__timeline_extraction", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 138.61336493492126, + "token_usage": { + "input_tokens": 58727, + "output_tokens": 2446, + "total_tokens": 61173, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 161.50582480430603, + "token_usage": { + "input_tokens": 127078, + "output_tokens": 2803, + "total_tokens": 129881, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 126.46522903442383, + "token_usage": { + "input_tokens": 100187, + "output_tokens": 2126, + "total_tokens": 102313, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 96.26100182533264, + "token_usage": { + "input_tokens": 56394, + "output_tokens": 2037, + "total_tokens": 58431, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/desktop_template__budget_computation.json b/model_results/deepseek-v3-1-terminus/desktop_template__budget_computation.json new file mode 100644 index 00000000..1817c3ee --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/desktop_template__budget_computation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "desktop_template__budget_computation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 132.58616971969604, + "token_usage": { + "input_tokens": 66967, + "output_tokens": 1777, + "total_tokens": 68744, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 122.725013256073, + "token_usage": { + "input_tokens": 51556, + "output_tokens": 1934, + "total_tokens": 53490, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 82.5663194656372, + "token_usage": { + "input_tokens": 44275, + "output_tokens": 1289, + "total_tokens": 45564, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 141.5936632156372, + "token_usage": { + "input_tokens": 82535, + "output_tokens": 2017, + "total_tokens": 84552, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/desktop_template__contact_information.json b/model_results/deepseek-v3-1-terminus/desktop_template__contact_information.json new file mode 100644 index 00000000..8e1952bf --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/desktop_template__contact_information.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "desktop_template__contact_information", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 149.86491012573242, + "token_usage": { + "input_tokens": 67225, + "output_tokens": 2234, + "total_tokens": 69459, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 144.40228056907654, + "token_usage": { + "input_tokens": 77463, + "output_tokens": 2134, + "total_tokens": 79597, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 11.23767614364624, + "token_usage": { + "input_tokens": 3187, + "output_tokens": 209, + "total_tokens": 3396, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 141.92485570907593, + "token_usage": { + "input_tokens": 53397, + "output_tokens": 2613, + "total_tokens": 56010, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/desktop_template__file_arrangement.json b/model_results/deepseek-v3-1-terminus/desktop_template__file_arrangement.json new file mode 100644 index 00000000..23e16845 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/desktop_template__file_arrangement.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "desktop_template__file_arrangement", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 444.45455861091614, + "token_usage": { + "input_tokens": 499497, + "output_tokens": 5039, + "total_tokens": 504536, + "reasoning_tokens": 0 + }, + "turn_count": 56 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 428.5998525619507, + "token_usage": { + "input_tokens": 487278, + "output_tokens": 5025, + "total_tokens": 492303, + "reasoning_tokens": 0 + }, + "turn_count": 55 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 248.51806163787842, + "token_usage": { + "input_tokens": 217679, + "output_tokens": 2867, + "total_tokens": 220546, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 223.56869792938232, + "token_usage": { + "input_tokens": 107906, + "output_tokens": 5192, + "total_tokens": 113098, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/dvdrental__customer_analysis_fix.json b/model_results/deepseek-v3-1-terminus/dvdrental__customer_analysis_fix.json new file mode 100644 index 00000000..5b3f8daa --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/dvdrental__customer_analysis_fix.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "dvdrental__customer_analysis_fix", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 304.07178139686584, + "token_usage": { + "input_tokens": 258500, + "output_tokens": 5163, + "total_tokens": 263663, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 311.8955788612366, + "token_usage": { + "input_tokens": 252658, + "output_tokens": 4708, + "total_tokens": 257366, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 344.65111804008484, + "token_usage": { + "input_tokens": 284849, + "output_tokens": 5986, + "total_tokens": 290835, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623580831380335966609074)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.938684225082397, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/dvdrental__customer_analytics_optimization.json b/model_results/deepseek-v3-1-terminus/dvdrental__customer_analytics_optimization.json new file mode 100644 index 00000000..963e75cc --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/dvdrental__customer_analytics_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "dvdrental__customer_analytics_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 239.05956530570984, + "token_usage": { + "input_tokens": 135124, + "output_tokens": 4276, + "total_tokens": 139400, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 199.89162397384644, + "token_usage": { + "input_tokens": 116717, + "output_tokens": 3357, + "total_tokens": 120074, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 152.3154890537262, + "token_usage": { + "input_tokens": 86817, + "output_tokens": 2598, + "total_tokens": 89415, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623581672243554401828671)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.180521011352539, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/dvdrental__film_inventory_management.json b/model_results/deepseek-v3-1-terminus/dvdrental__film_inventory_management.json new file mode 100644 index 00000000..1bbf6764 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/dvdrental__film_inventory_management.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "dvdrental__film_inventory_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 200.07974195480347, + "token_usage": { + "input_tokens": 227923, + "output_tokens": 2460, + "total_tokens": 230383, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 225.52838730812073, + "token_usage": { + "input_tokens": 195296, + "output_tokens": 3270, + "total_tokens": 198566, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 199.81371188163757, + "token_usage": { + "input_tokens": 202499, + "output_tokens": 2208, + "total_tokens": 204707, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 202509262358257978029672416304)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.201359987258911, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/easyr1__advanced_branch_strategy.json b/model_results/deepseek-v3-1-terminus/easyr1__advanced_branch_strategy.json new file mode 100644 index 00000000..d3c6d75e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/easyr1__advanced_branch_strategy.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "easyr1__advanced_branch_strategy", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 94.89696407318115, + "token_usage": { + "input_tokens": 145445, + "output_tokens": 1014, + "total_tokens": 146459, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622043576642227906441759)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.81865382194519, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 55.28738832473755, + "token_usage": { + "input_tokens": 23930, + "output_tokens": 838, + "total_tokens": 24768, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 86.20849013328552, + "token_usage": { + "input_tokens": 77371, + "output_tokens": 1566, + "total_tokens": 78937, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/easyr1__config_parameter_audit.json b/model_results/deepseek-v3-1-terminus/easyr1__config_parameter_audit.json new file mode 100644 index 00000000..3a916905 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/easyr1__config_parameter_audit.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "easyr1__config_parameter_audit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 252.89350628852844, + "token_usage": { + "input_tokens": 977681, + "output_tokens": 1940, + "total_tokens": 979621, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622063448989370545549454)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.738064527511597, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 727.9544639587402, + "token_usage": { + "input_tokens": 1190015, + "output_tokens": 1609, + "total_tokens": 1191624, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 607.4092285633087, + "token_usage": { + "input_tokens": 1315747, + "output_tokens": 2155, + "total_tokens": 1317902, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/easyr1__performance_regression_investigation.json b/model_results/deepseek-v3-1-terminus/easyr1__performance_regression_investigation.json new file mode 100644 index 00000000..dbeb4fa5 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/easyr1__performance_regression_investigation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "easyr1__performance_regression_investigation", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 216.1938133239746, + "token_usage": { + "input_tokens": 366322, + "output_tokens": 2752, + "total_tokens": 369074, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622082956503197275908153)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.795209407806396, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 212.5976676940918, + "token_usage": { + "input_tokens": 163507, + "output_tokens": 3188, + "total_tokens": 166695, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 225.86718153953552, + "token_usage": { + "input_tokens": 193134, + "output_tokens": 3990, + "total_tokens": 197124, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/easyr1__qwen3_issue_management.json b/model_results/deepseek-v3-1-terminus/easyr1__qwen3_issue_management.json new file mode 100644 index 00000000..b6d16052 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/easyr1__qwen3_issue_management.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "easyr1__qwen3_issue_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 96.25566291809082, + "token_usage": { + "input_tokens": 177209, + "output_tokens": 948, + "total_tokens": 178157, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 92.95255088806152, + "token_usage": { + "input_tokens": 162333, + "output_tokens": 872, + "total_tokens": 163205, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 99.37275528907776, + "token_usage": { + "input_tokens": 127991, + "output_tokens": 665, + "total_tokens": 128656, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 50.10952401161194, + "token_usage": { + "input_tokens": 101975, + "output_tokens": 560, + "total_tokens": 102535, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/employees__employee_demographics_report.json b/model_results/deepseek-v3-1-terminus/employees__employee_demographics_report.json new file mode 100644 index 00000000..9b9ca2fc --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/employees__employee_demographics_report.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "employees__employee_demographics_report", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 177.2384054660797, + "token_usage": { + "input_tokens": 126415, + "output_tokens": 2374, + "total_tokens": 128789, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 221.10795068740845, + "token_usage": { + "input_tokens": 137479, + "output_tokens": 3172, + "total_tokens": 140651, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 256.72964096069336, + "token_usage": { + "input_tokens": 152656, + "output_tokens": 3932, + "total_tokens": 156588, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623583547931144442139663)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.208264112472534, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/employees__employee_performance_analysis.json b/model_results/deepseek-v3-1-terminus/employees__employee_performance_analysis.json new file mode 100644 index 00000000..1dc5081d --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/employees__employee_performance_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "employees__employee_performance_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 209.56854391098022, + "token_usage": { + "input_tokens": 129164, + "output_tokens": 3086, + "total_tokens": 132250, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 330.1731457710266, + "token_usage": { + "input_tokens": 191625, + "output_tokens": 5463, + "total_tokens": 197088, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 191.89989256858826, + "token_usage": { + "input_tokens": 122804, + "output_tokens": 2684, + "total_tokens": 125488, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623584512780798899641300)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.182923793792725, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/employees__employee_project_tracking.json b/model_results/deepseek-v3-1-terminus/employees__employee_project_tracking.json new file mode 100644 index 00000000..ee0a0d04 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/employees__employee_project_tracking.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "employees__employee_project_tracking", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 254.79966402053833, + "token_usage": { + "input_tokens": 279479, + "output_tokens": 3292, + "total_tokens": 282771, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 283.3107876777649, + "token_usage": { + "input_tokens": 231103, + "output_tokens": 3711, + "total_tokens": 234814, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 244.51415610313416, + "token_usage": { + "input_tokens": 233443, + "output_tokens": 2865, + "total_tokens": 236308, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623585480766141949454287)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.218586683273315, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/employees__employee_retention_analysis.json b/model_results/deepseek-v3-1-terminus/employees__employee_retention_analysis.json new file mode 100644 index 00000000..1fc6243e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/employees__employee_retention_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "employees__employee_retention_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 349.061621427536, + "token_usage": { + "input_tokens": 290228, + "output_tokens": 5512, + "total_tokens": 295740, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 264.4508104324341, + "token_usage": { + "input_tokens": 183875, + "output_tokens": 3991, + "total_tokens": 187866, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 298.1451494693756, + "token_usage": { + "input_tokens": 234176, + "output_tokens": 4348, + "total_tokens": 238524, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623590447774066901206520)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.207370042800903, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/employees__executive_dashboard_automation.json b/model_results/deepseek-v3-1-terminus/employees__executive_dashboard_automation.json new file mode 100644 index 00000000..9897c911 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/employees__executive_dashboard_automation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "employees__executive_dashboard_automation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 390.2959725856781, + "token_usage": { + "input_tokens": 438405, + "output_tokens": 5526, + "total_tokens": 443931, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 359.1146535873413, + "token_usage": { + "input_tokens": 300441, + "output_tokens": 4498, + "total_tokens": 304939, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 283.06897473335266, + "token_usage": { + "input_tokens": 285543, + "output_tokens": 3465, + "total_tokens": 289008, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623591416350760863630366)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.23619031906128, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/employees__management_structure_analysis.json b/model_results/deepseek-v3-1-terminus/employees__management_structure_analysis.json new file mode 100644 index 00000000..ef428806 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/employees__management_structure_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "employees__management_structure_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 225.8659896850586, + "token_usage": { + "input_tokens": 235075, + "output_tokens": 3420, + "total_tokens": 238495, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 189.64699697494507, + "token_usage": { + "input_tokens": 165854, + "output_tokens": 2568, + "total_tokens": 168422, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 208.36173677444458, + "token_usage": { + "input_tokens": 185316, + "output_tokens": 2734, + "total_tokens": 188050, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623592392302501151301730)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.230656862258911, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/eval_web__cloudflare_turnstile_challenge.json b/model_results/deepseek-v3-1-terminus/eval_web__cloudflare_turnstile_challenge.json new file mode 100644 index 00000000..35a42471 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/eval_web__cloudflare_turnstile_challenge.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "eval_web__cloudflare_turnstile_challenge", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 340.6907594203949, + "token_usage": { + "input_tokens": 343102, + "output_tokens": 5100, + "total_tokens": 348202, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 355.706668138504, + "token_usage": { + "input_tokens": 294153, + "output_tokens": 5915, + "total_tokens": 300068, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 318.0956618785858, + "token_usage": { + "input_tokens": 301839, + "output_tokens": 5175, + "total_tokens": 307014, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 387.7790365219116, + "token_usage": { + "input_tokens": 403258, + "output_tokens": 4783, + "total_tokens": 408041, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/eval_web__extraction_table.json b/model_results/deepseek-v3-1-terminus/eval_web__extraction_table.json new file mode 100644 index 00000000..0d80c638 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/eval_web__extraction_table.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "eval_web__extraction_table", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 141.41911959648132, + "token_usage": { + "input_tokens": 150154, + "output_tokens": 2047, + "total_tokens": 152201, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 506.0817856788635, + "token_usage": { + "input_tokens": 1369313, + "output_tokens": 7514, + "total_tokens": 1376827, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 311.383891582489, + "token_usage": { + "input_tokens": 677156, + "output_tokens": 4514, + "total_tokens": 681670, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 367.910715341568, + "token_usage": { + "input_tokens": 677234, + "output_tokens": 4859, + "total_tokens": 682093, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/file_context__duplicates_searching.json b/model_results/deepseek-v3-1-terminus/file_context__duplicates_searching.json new file mode 100644 index 00000000..3acf2fbe --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/file_context__duplicates_searching.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "file_context__duplicates_searching", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 156.33373355865479, + "token_usage": { + "input_tokens": 82328, + "output_tokens": 2312, + "total_tokens": 84640, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 148.13799571990967, + "token_usage": { + "input_tokens": 73977, + "output_tokens": 2198, + "total_tokens": 76175, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 13.01108980178833, + "token_usage": { + "input_tokens": 3033, + "output_tokens": 172, + "total_tokens": 3205, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 97.14682984352112, + "token_usage": { + "input_tokens": 34981, + "output_tokens": 2098, + "total_tokens": 37079, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/file_context__file_merging.json b/model_results/deepseek-v3-1-terminus/file_context__file_merging.json new file mode 100644 index 00000000..391b3c31 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/file_context__file_merging.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "file_context__file_merging", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 82.26133799552917, + "token_usage": { + "input_tokens": 23533, + "output_tokens": 1363, + "total_tokens": 24896, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 161.9977617263794, + "token_usage": { + "input_tokens": 60378, + "output_tokens": 2733, + "total_tokens": 63111, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 121.05527997016907, + "token_usage": { + "input_tokens": 37450, + "output_tokens": 2511, + "total_tokens": 39961, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 59.1451141834259, + "token_usage": { + "input_tokens": 19170, + "output_tokens": 1064, + "total_tokens": 20234, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/file_context__file_splitting.json b/model_results/deepseek-v3-1-terminus/file_context__file_splitting.json new file mode 100644 index 00000000..85c7690f --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/file_context__file_splitting.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "file_context__file_splitting", + "runs": { + "run-1": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 1337 (char 1336)", + "execution_time": 55.424028158187866, + "token_usage": { + "input_tokens": 19435, + "output_tokens": 671, + "total_tokens": 20106, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 1337 (char 1336)", + "execution_time": 54.62150979042053, + "token_usage": { + "input_tokens": 19412, + "output_tokens": 660, + "total_tokens": 20072, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 150.56407022476196, + "token_usage": { + "input_tokens": 43327, + "output_tokens": 3035, + "total_tokens": 46362, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 62.24797868728638, + "token_usage": { + "input_tokens": 23752, + "output_tokens": 1211, + "total_tokens": 24963, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/file_context__pattern_matching.json b/model_results/deepseek-v3-1-terminus/file_context__pattern_matching.json new file mode 100644 index 00000000..d6224655 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/file_context__pattern_matching.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "file_context__pattern_matching", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 119.15594458580017, + "token_usage": { + "input_tokens": 38214, + "output_tokens": 1976, + "total_tokens": 40190, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 144.82859301567078, + "token_usage": { + "input_tokens": 64496, + "output_tokens": 2165, + "total_tokens": 66661, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 70.72907853126526, + "token_usage": { + "input_tokens": 24657, + "output_tokens": 967, + "total_tokens": 25624, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 66.93589568138123, + "token_usage": { + "input_tokens": 35601, + "output_tokens": 1196, + "total_tokens": 36797, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/file_context__uppercase.json b/model_results/deepseek-v3-1-terminus/file_context__uppercase.json new file mode 100644 index 00000000..27e8b177 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/file_context__uppercase.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "file_context__uppercase", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 161.1598241329193, + "token_usage": { + "input_tokens": 81410, + "output_tokens": 2271, + "total_tokens": 83681, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 161.52420949935913, + "token_usage": { + "input_tokens": 80667, + "output_tokens": 2019, + "total_tokens": 82686, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 102.55107808113098, + "token_usage": { + "input_tokens": 38517, + "output_tokens": 1981, + "total_tokens": 40498, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 100.03137135505676, + "token_usage": { + "input_tokens": 34960, + "output_tokens": 2057, + "total_tokens": 37017, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/file_property__size_classification.json b/model_results/deepseek-v3-1-terminus/file_property__size_classification.json new file mode 100644 index 00000000..fa1a2eba --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/file_property__size_classification.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "file_property__size_classification", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 120.33503413200378, + "token_usage": { + "input_tokens": 35190, + "output_tokens": 2085, + "total_tokens": 37275, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 122.93121337890625, + "token_usage": { + "input_tokens": 35115, + "output_tokens": 2033, + "total_tokens": 37148, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 133.4520697593689, + "token_usage": { + "input_tokens": 67018, + "output_tokens": 2212, + "total_tokens": 69230, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 5.349783182144165, + "token_usage": { + "input_tokens": 3026, + "output_tokens": 57, + "total_tokens": 3083, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/file_property__time_classification.json b/model_results/deepseek-v3-1-terminus/file_property__time_classification.json new file mode 100644 index 00000000..46a0b461 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/file_property__time_classification.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "file_property__time_classification", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 184.8964228630066, + "token_usage": { + "input_tokens": 87055, + "output_tokens": 3014, + "total_tokens": 90069, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 135.25379538536072, + "token_usage": { + "input_tokens": 58749, + "output_tokens": 2003, + "total_tokens": 60752, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 159.91613221168518, + "token_usage": { + "input_tokens": 46993, + "output_tokens": 3427, + "total_tokens": 50420, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 151.25762367248535, + "token_usage": { + "input_tokens": 77032, + "output_tokens": 2336, + "total_tokens": 79368, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/folder_structure__structure_analysis.json b/model_results/deepseek-v3-1-terminus/folder_structure__structure_analysis.json new file mode 100644 index 00000000..1d81188e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/folder_structure__structure_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "folder_structure__structure_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 146.94360637664795, + "token_usage": { + "input_tokens": 215097, + "output_tokens": 1574, + "total_tokens": 216671, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 138.36178493499756, + "token_usage": { + "input_tokens": 307052, + "output_tokens": 1531, + "total_tokens": 308583, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 53.85977840423584, + "token_usage": { + "input_tokens": 38737, + "output_tokens": 674, + "total_tokens": 39411, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 73.83413124084473, + "token_usage": { + "input_tokens": 118718, + "output_tokens": 715, + "total_tokens": 119433, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/folder_structure__structure_mirror.json b/model_results/deepseek-v3-1-terminus/folder_structure__structure_mirror.json new file mode 100644 index 00000000..76d3392a --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/folder_structure__structure_mirror.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "folder_structure__structure_mirror", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 369.82984828948975, + "token_usage": { + "input_tokens": 436585, + "output_tokens": 4163, + "total_tokens": 440748, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 342.6927993297577, + "token_usage": { + "input_tokens": 371846, + "output_tokens": 4118, + "total_tokens": 375964, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 306.3251585960388, + "token_usage": { + "input_tokens": 263742, + "output_tokens": 2713, + "total_tokens": 266455, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 114.73742127418518, + "token_usage": { + "input_tokens": 84609, + "output_tokens": 2084, + "total_tokens": 86693, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/harmony__fix_conflict.json b/model_results/deepseek-v3-1-terminus/harmony__fix_conflict.json new file mode 100644 index 00000000..c027f738 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/harmony__fix_conflict.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "harmony__fix_conflict", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 293.03666639328003, + "token_usage": { + "input_tokens": 1406576, + "output_tokens": 2583, + "total_tokens": 1409159, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622112371407575048270267)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.470372438430786, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 768.0327546596527, + "token_usage": { + "input_tokens": 1302560, + "output_tokens": 2559, + "total_tokens": 1305119, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": "Expecting ':' delimiter: line 1 column 482 (char 481)", + "execution_time": 245.54946327209473, + "token_usage": { + "input_tokens": 573912, + "output_tokens": 979, + "total_tokens": 574891, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/harmony__issue_pr_commit_workflow.json b/model_results/deepseek-v3-1-terminus/harmony__issue_pr_commit_workflow.json new file mode 100644 index 00000000..4b8845a8 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/harmony__issue_pr_commit_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "harmony__issue_pr_commit_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 163.6453673839569, + "token_usage": { + "input_tokens": 191791, + "output_tokens": 2299, + "total_tokens": 194090, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622122364134229088196195)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 9.889934539794922, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 205.85223817825317, + "token_usage": { + "input_tokens": 263992, + "output_tokens": 2187, + "total_tokens": 266179, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 209.96171307563782, + "token_usage": { + "input_tokens": 316421, + "output_tokens": 2607, + "total_tokens": 319028, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/harmony__issue_tagging_pr_closure.json b/model_results/deepseek-v3-1-terminus/harmony__issue_tagging_pr_closure.json new file mode 100644 index 00000000..83b643ed --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/harmony__issue_tagging_pr_closure.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "harmony__issue_tagging_pr_closure", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 222.64552283287048, + "token_usage": { + "input_tokens": 333265, + "output_tokens": 2954, + "total_tokens": 336219, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622132545900260604823051)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 12.07874083518982, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 297.49602937698364, + "token_usage": { + "input_tokens": 324974, + "output_tokens": 2858, + "total_tokens": 327832, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 143.3657112121582, + "token_usage": { + "input_tokens": 56346, + "output_tokens": 3310, + "total_tokens": 59656, + "reasoning_tokens": 0 + }, + "turn_count": 2 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/harmony__multi_branch_commit_aggregation.json b/model_results/deepseek-v3-1-terminus/harmony__multi_branch_commit_aggregation.json new file mode 100644 index 00000000..5148a8ee --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/harmony__multi_branch_commit_aggregation.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "harmony__multi_branch_commit_aggregation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 285.6013243198395, + "token_usage": { + "input_tokens": 655275, + "output_tokens": 3313, + "total_tokens": 658588, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622142484634635425480158)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.802415370941162, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 297.0952076911926, + "token_usage": { + "input_tokens": 286255, + "output_tokens": 3605, + "total_tokens": 289860, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 184.97724151611328, + "token_usage": { + "input_tokens": 249974, + "output_tokens": 3205, + "total_tokens": 253179, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/harmony__release_management_workflow.json b/model_results/deepseek-v3-1-terminus/harmony__release_management_workflow.json new file mode 100644 index 00000000..b6460eb9 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/harmony__release_management_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "harmony__release_management_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 614.7089338302612, + "token_usage": { + "input_tokens": 1503438, + "output_tokens": 10493, + "total_tokens": 1513931, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622152741232196234890688)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 9.933528900146484, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 730.019784450531, + "token_usage": { + "input_tokens": 1302166, + "output_tokens": 2883, + "total_tokens": 1305049, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 311.3378858566284, + "token_usage": { + "input_tokens": 331360, + "output_tokens": 5289, + "total_tokens": 336649, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/it_trouble_shooting_hub__asset_retirement_migration.json b/model_results/deepseek-v3-1-terminus/it_trouble_shooting_hub__asset_retirement_migration.json new file mode 100644 index 00000000..378b8642 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/it_trouble_shooting_hub__asset_retirement_migration.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "it_trouble_shooting_hub__asset_retirement_migration", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 371.3148522377014, + "token_usage": { + "input_tokens": 1174379, + "output_tokens": 4356, + "total_tokens": 1178735, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 273.0747570991516, + "token_usage": { + "input_tokens": 514753, + "output_tokens": 2589, + "total_tokens": 517342, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 208.73402976989746, + "token_usage": { + "input_tokens": 539875, + "output_tokens": 2028, + "total_tokens": 541903, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 188.8012306690216, + "token_usage": { + "input_tokens": 497125, + "output_tokens": 2047, + "total_tokens": 499172, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/it_trouble_shooting_hub__security_audit_ticket.json b/model_results/deepseek-v3-1-terminus/it_trouble_shooting_hub__security_audit_ticket.json new file mode 100644 index 00000000..9f83d53d --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/it_trouble_shooting_hub__security_audit_ticket.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "it_trouble_shooting_hub__security_audit_ticket", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 196.57794880867004, + "token_usage": { + "input_tokens": 635734, + "output_tokens": 2115, + "total_tokens": 637849, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 225.43587398529053, + "token_usage": { + "input_tokens": 438608, + "output_tokens": 2510, + "total_tokens": 441118, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 318.1488287448883, + "token_usage": { + "input_tokens": 1840410, + "output_tokens": 2585, + "total_tokens": 1842995, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 186.78525686264038, + "token_usage": { + "input_tokens": 527937, + "output_tokens": 1913, + "total_tokens": 529850, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/it_trouble_shooting_hub__verification_expired_update.json b/model_results/deepseek-v3-1-terminus/it_trouble_shooting_hub__verification_expired_update.json new file mode 100644 index 00000000..606c9ee6 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/it_trouble_shooting_hub__verification_expired_update.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "it_trouble_shooting_hub__verification_expired_update", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 229.99783205986023, + "token_usage": { + "input_tokens": 766405, + "output_tokens": 2391, + "total_tokens": 768796, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 140.7087812423706, + "token_usage": { + "input_tokens": 301093, + "output_tokens": 1276, + "total_tokens": 302369, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": "Extra data: line 1 column 520 (char 519)", + "execution_time": 203.37706303596497, + "token_usage": { + "input_tokens": 625154, + "output_tokens": 2019, + "total_tokens": 627173, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 160.08109521865845, + "token_usage": { + "input_tokens": 460592, + "output_tokens": 1725, + "total_tokens": 462317, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/japan_travel_planner__daily_itinerary_overview.json b/model_results/deepseek-v3-1-terminus/japan_travel_planner__daily_itinerary_overview.json new file mode 100644 index 00000000..2c380ee2 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/japan_travel_planner__daily_itinerary_overview.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "japan_travel_planner__daily_itinerary_overview", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 246.66852021217346, + "token_usage": { + "input_tokens": 771114, + "output_tokens": 2888, + "total_tokens": 774002, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 2590 (char 2589)", + "execution_time": 99.9729573726654, + "token_usage": { + "input_tokens": 105094, + "output_tokens": 1717, + "total_tokens": 106811, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 400.91640996932983, + "token_usage": { + "input_tokens": 1124434, + "output_tokens": 4515, + "total_tokens": 1128949, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 488.61013412475586, + "token_usage": { + "input_tokens": 2441300, + "output_tokens": 4978, + "total_tokens": 2446278, + "reasoning_tokens": 0 + }, + "turn_count": 31 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/japan_travel_planner__packing_progress_summary.json b/model_results/deepseek-v3-1-terminus/japan_travel_planner__packing_progress_summary.json new file mode 100644 index 00000000..9d5fca5b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/japan_travel_planner__packing_progress_summary.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "japan_travel_planner__packing_progress_summary", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 313.274475812912, + "token_usage": { + "input_tokens": 1510864, + "output_tokens": 2414, + "total_tokens": 1513278, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 129.94551348686218, + "token_usage": { + "input_tokens": 232836, + "output_tokens": 1535, + "total_tokens": 234371, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 292.5775797367096, + "token_usage": { + "input_tokens": 1267998, + "output_tokens": 2074, + "total_tokens": 1270072, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 287.5447256565094, + "token_usage": { + "input_tokens": 1323449, + "output_tokens": 2059, + "total_tokens": 1325508, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/japan_travel_planner__remove_osaka_itinerary.json b/model_results/deepseek-v3-1-terminus/japan_travel_planner__remove_osaka_itinerary.json new file mode 100644 index 00000000..6378408c --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/japan_travel_planner__remove_osaka_itinerary.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "japan_travel_planner__remove_osaka_itinerary", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 90.00246691703796, + "token_usage": { + "input_tokens": 137378, + "output_tokens": 975, + "total_tokens": 138353, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 173.94200658798218, + "token_usage": { + "input_tokens": 345204, + "output_tokens": 803, + "total_tokens": 346007, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 130.4261281490326, + "token_usage": { + "input_tokens": 431315, + "output_tokens": 1072, + "total_tokens": 432387, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 158.96713852882385, + "token_usage": { + "input_tokens": 485797, + "output_tokens": 1347, + "total_tokens": 487144, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/japan_travel_planner__restaurant_expenses_sync.json b/model_results/deepseek-v3-1-terminus/japan_travel_planner__restaurant_expenses_sync.json new file mode 100644 index 00000000..6551e882 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/japan_travel_planner__restaurant_expenses_sync.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "japan_travel_planner__restaurant_expenses_sync", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 256.4503297805786, + "token_usage": { + "input_tokens": 974271, + "output_tokens": 2840, + "total_tokens": 977111, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 252.6647515296936, + "token_usage": { + "input_tokens": 262755, + "output_tokens": 5073, + "total_tokens": 267828, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": "Extra data: line 1 column 398 (char 397)", + "execution_time": 49.02317476272583, + "token_usage": { + "input_tokens": 53790, + "output_tokens": 460, + "total_tokens": 54250, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 235.77568244934082, + "token_usage": { + "input_tokens": 433179, + "output_tokens": 2555, + "total_tokens": 435734, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/legal_document__dispute_review.json b/model_results/deepseek-v3-1-terminus/legal_document__dispute_review.json new file mode 100644 index 00000000..1ede1aad --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/legal_document__dispute_review.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "legal_document__dispute_review", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 78.72651314735413, + "token_usage": { + "input_tokens": 197089, + "output_tokens": 650, + "total_tokens": 197739, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 86.39774417877197, + "token_usage": { + "input_tokens": 134922, + "output_tokens": 982, + "total_tokens": 135904, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 114.42466068267822, + "token_usage": { + "input_tokens": 139796, + "output_tokens": 614, + "total_tokens": 140410, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 126.41040921211243, + "token_usage": { + "input_tokens": 140322, + "output_tokens": 1224, + "total_tokens": 141546, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/legal_document__individual_comments.json b/model_results/deepseek-v3-1-terminus/legal_document__individual_comments.json new file mode 100644 index 00000000..5e804055 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/legal_document__individual_comments.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "legal_document__individual_comments", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 140.80630946159363, + "token_usage": { + "input_tokens": 175736, + "output_tokens": 1926, + "total_tokens": 177662, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 238.32636213302612, + "token_usage": { + "input_tokens": 1172588, + "output_tokens": 2636, + "total_tokens": 1175224, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": "Extra data: line 1 column 281 (char 280)", + "execution_time": 119.4811179637909, + "token_usage": { + "input_tokens": 96756, + "output_tokens": 1326, + "total_tokens": 98082, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": "Extra data: line 1 column 289 (char 288)", + "execution_time": 86.98640394210815, + "token_usage": { + "input_tokens": 100961, + "output_tokens": 607, + "total_tokens": 101568, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/legal_document__solution_tracing.json b/model_results/deepseek-v3-1-terminus/legal_document__solution_tracing.json new file mode 100644 index 00000000..6d9af678 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/legal_document__solution_tracing.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "legal_document__solution_tracing", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 132.0972511768341, + "token_usage": { + "input_tokens": 215309, + "output_tokens": 1297, + "total_tokens": 216606, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 172.6146595478058, + "token_usage": { + "input_tokens": 356709, + "output_tokens": 1544, + "total_tokens": 358253, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 267.18336248397827, + "token_usage": { + "input_tokens": 322942, + "output_tokens": 1151, + "total_tokens": 324093, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 373.6008851528168, + "token_usage": { + "input_tokens": 419197, + "output_tokens": 2241, + "total_tokens": 421438, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/lego__consistency_enforcement.json b/model_results/deepseek-v3-1-terminus/lego__consistency_enforcement.json new file mode 100644 index 00000000..aa7089f4 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/lego__consistency_enforcement.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "lego__consistency_enforcement", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 332.5333845615387, + "token_usage": { + "input_tokens": 1315257, + "output_tokens": 4759, + "total_tokens": 1320016, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 199974 tokens (199974 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092615465284190712441372342)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 189.04312682151794, + "token_usage": { + "input_tokens": 444134, + "output_tokens": 1721, + "total_tokens": 445855, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 640.5812239646912, + "token_usage": { + "input_tokens": 1399627, + "output_tokens": 5161, + "total_tokens": 1404788, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623593245679409635300235)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.20232081413269, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/lego__database_security_policies.json b/model_results/deepseek-v3-1-terminus/lego__database_security_policies.json new file mode 100644 index 00000000..57e4fa4b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/lego__database_security_policies.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "lego__database_security_policies", + "runs": { + "run-1": { + "success": true, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 546050 tokens (546050 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 195.53896188735962, + "token_usage": { + "input_tokens": 172815, + "output_tokens": 2186, + "total_tokens": 175001, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 802.7101278305054, + "token_usage": { + "input_tokens": 2309536, + "output_tokens": 10350, + "total_tokens": 2319886, + "reasoning_tokens": 0 + }, + "turn_count": 60 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 301.2383608818054, + "token_usage": { + "input_tokens": 228041, + "output_tokens": 4329, + "total_tokens": 232370, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623594132100573885226991)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.234580516815186, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/lego__transactional_inventory_transfer.json b/model_results/deepseek-v3-1-terminus/lego__transactional_inventory_transfer.json new file mode 100644 index 00000000..a5349672 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/lego__transactional_inventory_transfer.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "lego__transactional_inventory_transfer", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 736.6222898960114, + "token_usage": { + "input_tokens": 683489, + "output_tokens": 15041, + "total_tokens": 698530, + "reasoning_tokens": 0 + }, + "turn_count": 43 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 673.1510260105133, + "token_usage": { + "input_tokens": 671371, + "output_tokens": 11592, + "total_tokens": 682963, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 908.8019723892212, + "token_usage": { + "input_tokens": 927507, + "output_tokens": 16115, + "total_tokens": 943622, + "reasoning_tokens": 0 + }, + "turn_count": 49 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623594981107139665863566)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.176013946533203, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/mcpmark-cicd__deployment_status_workflow.json b/model_results/deepseek-v3-1-terminus/mcpmark-cicd__deployment_status_workflow.json new file mode 100644 index 00000000..17c247c4 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/mcpmark-cicd__deployment_status_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "mcpmark-cicd__deployment_status_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 233.63362741470337, + "token_usage": { + "input_tokens": 101102, + "output_tokens": 4806, + "total_tokens": 105908, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622154215815433883810580)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.76741623878479, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 249.43173933029175, + "token_usage": { + "input_tokens": 136980, + "output_tokens": 4722, + "total_tokens": 141702, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 184 (char 183)", + "execution_time": 113.7837073802948, + "token_usage": { + "input_tokens": 48735, + "output_tokens": 3097, + "total_tokens": 51832, + "reasoning_tokens": 0 + }, + "turn_count": 2 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/mcpmark-cicd__issue_management_workflow.json b/model_results/deepseek-v3-1-terminus/mcpmark-cicd__issue_management_workflow.json new file mode 100644 index 00000000..d6ceb64d --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/mcpmark-cicd__issue_management_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "mcpmark-cicd__issue_management_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 525.1548409461975, + "token_usage": { + "input_tokens": 1529635, + "output_tokens": 7300, + "total_tokens": 1536935, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622173169438181389109547)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 11.621798515319824, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 307.3473916053772, + "token_usage": { + "input_tokens": 226627, + "output_tokens": 4940, + "total_tokens": 231567, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 464.826598405838, + "token_usage": { + "input_tokens": 505330, + "output_tokens": 8633, + "total_tokens": 513963, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/mcpmark-cicd__linting_ci_workflow.json b/model_results/deepseek-v3-1-terminus/mcpmark-cicd__linting_ci_workflow.json new file mode 100644 index 00000000..9641cba2 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/mcpmark-cicd__linting_ci_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "mcpmark-cicd__linting_ci_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 117.18397498130798, + "token_usage": { + "input_tokens": 133327, + "output_tokens": 1592, + "total_tokens": 134919, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622191932417092542435156)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.769605875015259, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 118.1135323047638, + "token_usage": { + "input_tokens": 124956, + "output_tokens": 1322, + "total_tokens": 126278, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 14.015326023101807, + "token_usage": { + "input_tokens": 24033, + "output_tokens": 87, + "total_tokens": 24120, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/mcpmark-cicd__pr_automation_workflow.json b/model_results/deepseek-v3-1-terminus/mcpmark-cicd__pr_automation_workflow.json new file mode 100644 index 00000000..b46c9e59 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/mcpmark-cicd__pr_automation_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "mcpmark-cicd__pr_automation_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 287.7944805622101, + "token_usage": { + "input_tokens": 246758, + "output_tokens": 4898, + "total_tokens": 251656, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622193598305134280399336)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 11.336401224136353, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 17.65446400642395, + "token_usage": { + "input_tokens": 24006, + "output_tokens": 92, + "total_tokens": 24098, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 155.7432405948639, + "token_usage": { + "input_tokens": 133207, + "output_tokens": 3844, + "total_tokens": 137051, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/missing-semester__assign_contributor_labels.json b/model_results/deepseek-v3-1-terminus/missing-semester__assign_contributor_labels.json new file mode 100644 index 00000000..cf76499b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/missing-semester__assign_contributor_labels.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "missing-semester__assign_contributor_labels", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 235.4257414340973, + "token_usage": { + "input_tokens": 1264610, + "output_tokens": 1617, + "total_tokens": 1266227, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 20250926222312682000706712120)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 11.687310695648193, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 613.6650176048279, + "token_usage": { + "input_tokens": 1047892, + "output_tokens": 2133, + "total_tokens": 1050025, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 151.15459322929382, + "token_usage": { + "input_tokens": 380809, + "output_tokens": 1588, + "total_tokens": 382397, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/missing-semester__find_legacy_name.json b/model_results/deepseek-v3-1-terminus/missing-semester__find_legacy_name.json new file mode 100644 index 00000000..d8b9d112 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/missing-semester__find_legacy_name.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "missing-semester__find_legacy_name", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 137631 tokens (137631 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092618200112553890500668545)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 93.55011773109436, + "token_usage": { + "input_tokens": 247911, + "output_tokens": 263, + "total_tokens": 248174, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622244857166092602471937)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 10.70331597328186, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 170337 exceeds the maximum length 98304. Request id: 021758960681780096fce02f344d1557d64e35a80bc1dfd5772f0 (tid: 2025092708102537194135003863719)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 744.953742980957, + "token_usage": { + "input_tokens": 1089926, + "output_tokens": 790, + "total_tokens": 1090716, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": "Extra data: line 1 column 134 (char 133)", + "execution_time": 118.13037371635437, + "token_usage": { + "input_tokens": 331872, + "output_tokens": 489, + "total_tokens": 332361, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/missing-semester__find_salient_file.json b/model_results/deepseek-v3-1-terminus/missing-semester__find_salient_file.json new file mode 100644 index 00000000..2dbb76f6 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/missing-semester__find_salient_file.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "github", + "task": "missing-semester__find_salient_file", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 560.5716016292572, + "token_usage": { + "input_tokens": 2938725, + "output_tokens": 5036, + "total_tokens": 2943761, + "reasoning_tokens": 0 + }, + "turn_count": 50 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092622262021727691478321575)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 11.157562017440796, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 902.2717890739441, + "token_usage": { + "input_tokens": 1524753, + "output_tokens": 4662, + "total_tokens": 1529415, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 411.1900761127472, + "token_usage": { + "input_tokens": 1171924, + "output_tokens": 3430, + "total_tokens": 1175354, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/online_resume__layout_adjustment.json b/model_results/deepseek-v3-1-terminus/online_resume__layout_adjustment.json new file mode 100644 index 00000000..b96d7f6d --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/online_resume__layout_adjustment.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "online_resume__layout_adjustment", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 280.65002727508545, + "token_usage": { + "input_tokens": 821187, + "output_tokens": 2830, + "total_tokens": 824017, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 394.77181601524353, + "token_usage": { + "input_tokens": 1027774, + "output_tokens": 3167, + "total_tokens": 1030941, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 272.48287296295166, + "token_usage": { + "input_tokens": 836279, + "output_tokens": 2693, + "total_tokens": 838972, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 310.2703666687012, + "token_usage": { + "input_tokens": 980604, + "output_tokens": 2752, + "total_tokens": 983356, + "reasoning_tokens": 0 + }, + "turn_count": 29 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/online_resume__projects_section_update.json b/model_results/deepseek-v3-1-terminus/online_resume__projects_section_update.json new file mode 100644 index 00000000..71594adf --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/online_resume__projects_section_update.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "online_resume__projects_section_update", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 240.42282342910767, + "token_usage": { + "input_tokens": 679571, + "output_tokens": 2464, + "total_tokens": 682035, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 510 (char 509)", + "execution_time": 95.56528234481812, + "token_usage": { + "input_tokens": 226274, + "output_tokens": 929, + "total_tokens": 227203, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 235.71794629096985, + "token_usage": { + "input_tokens": 629808, + "output_tokens": 2408, + "total_tokens": 632216, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 305.6466438770294, + "token_usage": { + "input_tokens": 914592, + "output_tokens": 2838, + "total_tokens": 917430, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/online_resume__skills_development_tracker.json b/model_results/deepseek-v3-1-terminus/online_resume__skills_development_tracker.json new file mode 100644 index 00000000..01b998d5 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/online_resume__skills_development_tracker.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "online_resume__skills_development_tracker", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 370.6589341163635, + "token_usage": { + "input_tokens": 968800, + "output_tokens": 4270, + "total_tokens": 973070, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 11.19611644744873, + "token_usage": { + "input_tokens": 6710, + "output_tokens": 215, + "total_tokens": 6925, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 559.2584075927734, + "token_usage": { + "input_tokens": 2200059, + "output_tokens": 5726, + "total_tokens": 2205785, + "reasoning_tokens": 0 + }, + "turn_count": 48 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 295.5869219303131, + "token_usage": { + "input_tokens": 424892, + "output_tokens": 3350, + "total_tokens": 428242, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/online_resume__work_history_addition.json b/model_results/deepseek-v3-1-terminus/online_resume__work_history_addition.json new file mode 100644 index 00000000..1bf24ed5 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/online_resume__work_history_addition.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "online_resume__work_history_addition", + "runs": { + "run-1": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 1990 (char 1989)", + "execution_time": 211.33162832260132, + "token_usage": { + "input_tokens": 507721, + "output_tokens": 2507, + "total_tokens": 510228, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 449.900093793869, + "token_usage": { + "input_tokens": 554243, + "output_tokens": 9017, + "total_tokens": 563260, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-3": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 1601 (char 1600)", + "execution_time": 294.91173124313354, + "token_usage": { + "input_tokens": 1017031, + "output_tokens": 3163, + "total_tokens": 1020194, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": "Extra data: line 1 column 3141 (char 3140)", + "execution_time": 198.33850598335266, + "token_usage": { + "input_tokens": 287339, + "output_tokens": 2489, + "total_tokens": 289828, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/papers__author_folders.json b/model_results/deepseek-v3-1-terminus/papers__author_folders.json new file mode 100644 index 00000000..f6c2079f --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/papers__author_folders.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "papers__author_folders", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 1760651 tokens (1760651 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092616364191512760445520182)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 187.72283601760864, + "token_usage": { + "input_tokens": 65623, + "output_tokens": 3502, + "total_tokens": 69125, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 308947 tokens (308947 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092620345198190139357002282)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 63.74360632896423, + "token_usage": { + "input_tokens": 66822, + "output_tokens": 736, + "total_tokens": 67558, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 200860 exceeds the maximum length 98304. Request id: 021759027605309dd3a5faac23f0c9eed6bd811669e43cb37224f (tid: 2025092802460523882120781075485)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 266.04863381385803, + "token_usage": { + "input_tokens": 130506, + "output_tokens": 725, + "total_tokens": 131231, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input token exceeded (tid: 2025092802105831584151910886707)\",\"type\":\"upstream_error\",\"param\":\"400\",\"code\":\"bad_response_status_code\"}}", + "execution_time": 259.42858505249023, + "token_usage": { + "input_tokens": 11334, + "output_tokens": 545, + "total_tokens": 11879, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/papers__find_math_paper.json b/model_results/deepseek-v3-1-terminus/papers__find_math_paper.json new file mode 100644 index 00000000..7310b23b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/papers__find_math_paper.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "papers__find_math_paper", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 163789 tokens (163789 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092616392993690912670541264)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 160.2336859703064, + "token_usage": { + "input_tokens": 516390, + "output_tokens": 1067, + "total_tokens": 517457, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 146146 exceeds the maximum length 98304. Request id: 021759023276137e2db2f94dd3369291ecdb9ac35cb66923d1f7e (tid: 2025092801335466206118399164152)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 194.11977434158325, + "token_usage": { + "input_tokens": 105742, + "output_tokens": 680, + "total_tokens": 106422, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 147076 exceeds the maximum length 98304. Request id: 0217589538927080c6bbe1681ccb3bc505384f293d30c2f795969 (tid: 2025092706172944705974529006721)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 398.12489771842957, + "token_usage": { + "input_tokens": 387907, + "output_tokens": 834, + "total_tokens": 388741, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 160372 exceeds the maximum length 98304. Request id: 0217589922328409fc4dc47d27ebf7239e22958e08dac0ba73ed8 (tid: 2025092716561318844708817041337)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 229.76173663139343, + "token_usage": { + "input_tokens": 10140, + "output_tokens": 450, + "total_tokens": 10590, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/papers__organize_legacy_papers.json b/model_results/deepseek-v3-1-terminus/papers__organize_legacy_papers.json new file mode 100644 index 00000000..c3b2566b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/papers__organize_legacy_papers.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "papers__organize_legacy_papers", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 325223 tokens (325223 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092616420022789466676635775)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 150.26462125778198, + "token_usage": { + "input_tokens": 523809, + "output_tokens": 1220, + "total_tokens": 525029, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 323499 tokens (323499 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092620420949283332608068290)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 79.78834533691406, + "token_usage": { + "input_tokens": 85858, + "output_tokens": 830, + "total_tokens": 86688, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 6.471507787704468, + "token_usage": { + "input_tokens": 3559, + "output_tokens": 33, + "total_tokens": 3592, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input token exceeded (tid: 2025092717005597117451423883734)\",\"type\":\"upstream_error\",\"param\":\"400\",\"code\":\"bad_response_status_code\"}}", + "execution_time": 340.7232069969177, + "token_usage": { + "input_tokens": 11952, + "output_tokens": 555, + "total_tokens": 12507, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/python_roadmap__expert_level_lessons.json b/model_results/deepseek-v3-1-terminus/python_roadmap__expert_level_lessons.json new file mode 100644 index 00000000..2119301d --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/python_roadmap__expert_level_lessons.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "python_roadmap__expert_level_lessons", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 291.72464203834534, + "token_usage": { + "input_tokens": 1057888, + "output_tokens": 3011, + "total_tokens": 1060899, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 285.7454922199249, + "token_usage": { + "input_tokens": 703963, + "output_tokens": 3664, + "total_tokens": 707627, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 597.889726638794, + "token_usage": { + "input_tokens": 3241155, + "output_tokens": 5440, + "total_tokens": 3246595, + "reasoning_tokens": 0 + }, + "turn_count": 43 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 327.34903502464294, + "token_usage": { + "input_tokens": 1442443, + "output_tokens": 3443, + "total_tokens": 1445886, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/python_roadmap__learning_metrics_dashboard.json b/model_results/deepseek-v3-1-terminus/python_roadmap__learning_metrics_dashboard.json new file mode 100644 index 00000000..bb3b017e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/python_roadmap__learning_metrics_dashboard.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "python_roadmap__learning_metrics_dashboard", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 305.62830877304077, + "token_usage": { + "input_tokens": 1164054, + "output_tokens": 3044, + "total_tokens": 1167098, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 199.27596855163574, + "token_usage": { + "input_tokens": 411218, + "output_tokens": 2996, + "total_tokens": 414214, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 395.6280062198639, + "token_usage": { + "input_tokens": 1793209, + "output_tokens": 4069, + "total_tokens": 1797278, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 215.58000874519348, + "token_usage": { + "input_tokens": 310733, + "output_tokens": 3078, + "total_tokens": 313811, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/reddit__ai_data_analyst.json b/model_results/deepseek-v3-1-terminus/reddit__ai_data_analyst.json new file mode 100644 index 00000000..d0744522 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/reddit__ai_data_analyst.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "reddit__ai_data_analyst", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 134146 tokens (134146 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 228.44413709640503, + "token_usage": { + "input_tokens": 705107, + "output_tokens": 1087, + "total_tokens": 706194, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 140966 tokens (140966 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092619075644416868899775124)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 203.9344620704651, + "token_usage": { + "input_tokens": 501001, + "output_tokens": 965, + "total_tokens": 501966, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092701211472386915957817128)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 36.51499533653259, + "token_usage": { + "input_tokens": 19569, + "output_tokens": 268, + "total_tokens": 19837, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 131064 exceeds the maximum length 98304. Request id: 0217589758613457a20ba5082a436aed6033d266df24c7ee241e6 (tid: 2025092712232629324735662242053)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 564.9863498210907, + "token_usage": { + "input_tokens": 674826, + "output_tokens": 1256, + "total_tokens": 676082, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/reddit__budget_europe_travel.json b/model_results/deepseek-v3-1-terminus/reddit__budget_europe_travel.json new file mode 100644 index 00000000..758ba4ef --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/reddit__budget_europe_travel.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "reddit__budget_europe_travel", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 277.6534972190857, + "token_usage": { + "input_tokens": 818825, + "output_tokens": 2398, + "total_tokens": 821223, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 271.2076871395111, + "token_usage": { + "input_tokens": 742915, + "output_tokens": 2206, + "total_tokens": 745121, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 558.0852010250092, + "token_usage": { + "input_tokens": 857847, + "output_tokens": 2212, + "total_tokens": 860059, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 847.8150887489319, + "token_usage": { + "input_tokens": 1396152, + "output_tokens": 2480, + "total_tokens": 1398632, + "reasoning_tokens": 0 + }, + "turn_count": 39 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/reddit__buyitforlife_research.json b/model_results/deepseek-v3-1-terminus/reddit__buyitforlife_research.json new file mode 100644 index 00000000..11f239dd --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/reddit__buyitforlife_research.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "reddit__buyitforlife_research", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 149412 tokens (149412 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 119.06816720962524, + "token_usage": { + "input_tokens": 233113, + "output_tokens": 727, + "total_tokens": 233840, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 149443 tokens (149443 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092619160836131684715707205)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 120.34467816352844, + "token_usage": { + "input_tokens": 233768, + "output_tokens": 723, + "total_tokens": 234491, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 168594 exceeds the maximum length 98304. Request id: 02175893712160364d058e651d4e36992a0815355aa97dad3796c (tid: 2025092701375211808371388444164)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 392.5510013103485, + "token_usage": { + "input_tokens": 391180, + "output_tokens": 711, + "total_tokens": 391891, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 566.3692433834076, + "token_usage": { + "input_tokens": 868222, + "output_tokens": 1998, + "total_tokens": 870220, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/reddit__llm_research_summary.json b/model_results/deepseek-v3-1-terminus/reddit__llm_research_summary.json new file mode 100644 index 00000000..407044b7 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/reddit__llm_research_summary.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "reddit__llm_research_summary", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 133933 tokens (133933 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 236.75895142555237, + "token_usage": { + "input_tokens": 726255, + "output_tokens": 1749, + "total_tokens": 728004, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 300.7061505317688, + "token_usage": { + "input_tokens": 835872, + "output_tokens": 3337, + "total_tokens": 839209, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 12.010667562484741, + "token_usage": { + "input_tokens": 4437, + "output_tokens": 176, + "total_tokens": 4613, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 140580 exceeds the maximum length 98304. Request id: 0217589782691326c538cdb85280b87fd1d855e8af4abb8bfaa3b (tid: 2025092713034126559409445561118)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 842.587836265564, + "token_usage": { + "input_tokens": 971342, + "output_tokens": 2628, + "total_tokens": 973970, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/reddit__movie_reviewer_analysis.json b/model_results/deepseek-v3-1-terminus/reddit__movie_reviewer_analysis.json new file mode 100644 index 00000000..db7aa855 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/reddit__movie_reviewer_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "reddit__movie_reviewer_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 139246 tokens (139246 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 275.7007510662079, + "token_usage": { + "input_tokens": 674460, + "output_tokens": 2511, + "total_tokens": 676971, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 299.4474368095398, + "token_usage": { + "input_tokens": 800781, + "output_tokens": 3649, + "total_tokens": 804430, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 143808 exceeds the maximum length 98304. Request id: 0217589378891877fc34901b454b05ffbd36a7adad2e1996c303f (tid: 2025092701505583190905809800974)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 662.4497663974762, + "token_usage": { + "input_tokens": 692600, + "output_tokens": 3833, + "total_tokens": 696433, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 766.1712210178375, + "token_usage": { + "input_tokens": 1037160, + "output_tokens": 3055, + "total_tokens": 1040215, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/reddit__nba_statistics_analysis.json b/model_results/deepseek-v3-1-terminus/reddit__nba_statistics_analysis.json new file mode 100644 index 00000000..1a752fd6 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/reddit__nba_statistics_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "reddit__nba_statistics_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 133670 tokens (133670 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 281.4182059764862, + "token_usage": { + "input_tokens": 1192604, + "output_tokens": 1815, + "total_tokens": 1194419, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 137554 tokens (137554 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092619322145217105932124145)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 223.13858580589294, + "token_usage": { + "input_tokens": 541073, + "output_tokens": 1488, + "total_tokens": 542561, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 135089 exceeds the maximum length 98304. Request id: 021759013140244860709f8992798899d3f9babbf4f90d5a950bb (tid: 2025092722453573711098354392248)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 319.2330069541931, + "token_usage": { + "input_tokens": 852795, + "output_tokens": 2327, + "total_tokens": 855122, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 15.212673664093018, + "token_usage": { + "input_tokens": 4488, + "output_tokens": 190, + "total_tokens": 4678, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/reddit__routine_tracker_forum.json b/model_results/deepseek-v3-1-terminus/reddit__routine_tracker_forum.json new file mode 100644 index 00000000..a9ebd9b5 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/reddit__routine_tracker_forum.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "reddit__routine_tracker_forum", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 137670 tokens (137670 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 84.26762819290161, + "token_usage": { + "input_tokens": 108199, + "output_tokens": 610, + "total_tokens": 108809, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 133004 tokens (133004 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092619362385477492245711844)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 192.07586979866028, + "token_usage": { + "input_tokens": 456113, + "output_tokens": 1039, + "total_tokens": 457152, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 141699 exceeds the maximum length 98304. Request id: 0217589389583702dd0dd99c228a5c584388c2ba587875874d095 (tid: 2025092702083247160144345404567)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 292.9134404659271, + "token_usage": { + "input_tokens": 204877, + "output_tokens": 618, + "total_tokens": 205495, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 10.692319631576538, + "token_usage": { + "input_tokens": 4307, + "output_tokens": 189, + "total_tokens": 4496, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/security__rls_business_access.json b/model_results/deepseek-v3-1-terminus/security__rls_business_access.json new file mode 100644 index 00000000..2e6252f3 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/security__rls_business_access.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "security__rls_business_access", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 254.88371181488037, + "token_usage": { + "input_tokens": 205896, + "output_tokens": 4375, + "total_tokens": 210271, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 332.03830099105835, + "token_usage": { + "input_tokens": 267193, + "output_tokens": 5243, + "total_tokens": 272436, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 293.414235830307, + "token_usage": { + "input_tokens": 189868, + "output_tokens": 4662, + "total_tokens": 194530, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092623595850692997622132433)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.234999179840088, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/security__user_permission_audit.json b/model_results/deepseek-v3-1-terminus/security__user_permission_audit.json new file mode 100644 index 00000000..e09a35e4 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/security__user_permission_audit.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "security__user_permission_audit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 293.8720893859863, + "token_usage": { + "input_tokens": 191825, + "output_tokens": 5986, + "total_tokens": 197811, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 224.15211606025696, + "token_usage": { + "input_tokens": 128597, + "output_tokens": 3711, + "total_tokens": 132308, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 261.39845728874207, + "token_usage": { + "input_tokens": 147157, + "output_tokens": 4468, + "total_tokens": 151625, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092700000719241031516213195)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.308505773544312, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/self_assessment__faq_column_layout.json b/model_results/deepseek-v3-1-terminus/self_assessment__faq_column_layout.json new file mode 100644 index 00000000..4adaee95 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/self_assessment__faq_column_layout.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "self_assessment__faq_column_layout", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 336.40138387680054, + "token_usage": { + "input_tokens": 973443, + "output_tokens": 3126, + "total_tokens": 976569, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-2": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 1841 (char 1840)", + "execution_time": 77.71289849281311, + "token_usage": { + "input_tokens": 113533, + "output_tokens": 1094, + "total_tokens": 114627, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 200.61159801483154, + "token_usage": { + "input_tokens": 631844, + "output_tokens": 1993, + "total_tokens": 633837, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 206.68745589256287, + "token_usage": { + "input_tokens": 482174, + "output_tokens": 1804, + "total_tokens": 483978, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/self_assessment__hyperfocus_analysis_report.json b/model_results/deepseek-v3-1-terminus/self_assessment__hyperfocus_analysis_report.json new file mode 100644 index 00000000..fabf27ca --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/self_assessment__hyperfocus_analysis_report.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "self_assessment__hyperfocus_analysis_report", + "runs": { + "run-1": { + "success": false, + "error_message": "Expecting value: line 1 column 11604 (char 11603)", + "execution_time": 257.5691246986389, + "token_usage": { + "input_tokens": 211537, + "output_tokens": 4692, + "total_tokens": 216229, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 158.84276294708252, + "token_usage": { + "input_tokens": 262346, + "output_tokens": 2737, + "total_tokens": 265083, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 352.6119296550751, + "token_usage": { + "input_tokens": 495782, + "output_tokens": 5423, + "total_tokens": 501205, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 331.8921048641205, + "token_usage": { + "input_tokens": 836669, + "output_tokens": 4143, + "total_tokens": 840812, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/self_assessment__numbered_list_emojis.json b/model_results/deepseek-v3-1-terminus/self_assessment__numbered_list_emojis.json new file mode 100644 index 00000000..f75b4c2e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/self_assessment__numbered_list_emojis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "self_assessment__numbered_list_emojis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 403.21801567077637, + "token_usage": { + "input_tokens": 1665948, + "output_tokens": 3235, + "total_tokens": 1669183, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 73.6966302394867, + "token_usage": { + "input_tokens": 104814, + "output_tokens": 1474, + "total_tokens": 106288, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 382.89486956596375, + "token_usage": { + "input_tokens": 1718346, + "output_tokens": 3796, + "total_tokens": 1722142, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 102.92528820037842, + "token_usage": { + "input_tokens": 275552, + "output_tokens": 847, + "total_tokens": 276399, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping__advanced_product_analysis.json b/model_results/deepseek-v3-1-terminus/shopping__advanced_product_analysis.json new file mode 100644 index 00000000..f44f893f --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping__advanced_product_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping__advanced_product_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 140517 tokens (140517 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 245.26102876663208, + "token_usage": { + "input_tokens": 930637, + "output_tokens": 1242, + "total_tokens": 931879, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 137092 tokens (137092 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092619413847404357633461690)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 243.48658752441406, + "token_usage": { + "input_tokens": 891158, + "output_tokens": 1333, + "total_tokens": 892491, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 130133 exceeds the maximum length 98304. Request id: 021758939846598ba5ee286f4f61d26104ad1883c800bc0a6520e (tid: 202509270222572549309122235586)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 819.8968021869659, + "token_usage": { + "input_tokens": 1002016, + "output_tokens": 2160, + "total_tokens": 1004176, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 130943 exceeds the maximum length 98304. Request id: 02175898049923230b25afe6d7c40e538a44e9b9be5be807aef64 (tid: 202509271340131625742165772330)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 1232.4235219955444, + "token_usage": { + "input_tokens": 1378068, + "output_tokens": 3012, + "total_tokens": 1381080, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping__gaming_accessories_analysis.json b/model_results/deepseek-v3-1-terminus/shopping__gaming_accessories_analysis.json new file mode 100644 index 00000000..12c5c975 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping__gaming_accessories_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping__gaming_accessories_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 132903 tokens (132903 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 419.0592272281647, + "token_usage": { + "input_tokens": 2072505, + "output_tokens": 3585, + "total_tokens": 2076090, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 132145 tokens (132145 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092619493790190887528905414)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 416.41197776794434, + "token_usage": { + "input_tokens": 2309633, + "output_tokens": 2960, + "total_tokens": 2312593, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 932.1683664321899, + "token_usage": { + "input_tokens": 1363231, + "output_tokens": 3914, + "total_tokens": 1367145, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": "Extra data: line 1 column 1891 (char 1890)", + "execution_time": 67.24101996421814, + "token_usage": { + "input_tokens": 39376, + "output_tokens": 882, + "total_tokens": 40258, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping__health_routine_optimization.json b/model_results/deepseek-v3-1-terminus/shopping__health_routine_optimization.json new file mode 100644 index 00000000..eac2b20f --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping__health_routine_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping__health_routine_optimization", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 136403 tokens (136403 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 263.12338614463806, + "token_usage": { + "input_tokens": 992238, + "output_tokens": 1455, + "total_tokens": 993693, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 139434 tokens (139434 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092619554430583018182591703)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 303.2782862186432, + "token_usage": { + "input_tokens": 1426931, + "output_tokens": 1563, + "total_tokens": 1428494, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 789.2739129066467, + "token_usage": { + "input_tokens": 1433901, + "output_tokens": 1234, + "total_tokens": 1435135, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 137254 exceeds the maximum length 98304. Request id: 0217589821049377252df8a07a29d0ed7f8728b32b20a065a53d2 (tid: 202509271407273937150386526475)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 1412.375020980835, + "token_usage": { + "input_tokens": 1635582, + "output_tokens": 2448, + "total_tokens": 1638030, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping__holiday_baking_competition.json b/model_results/deepseek-v3-1-terminus/shopping__holiday_baking_competition.json new file mode 100644 index 00000000..45a2f4f5 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping__holiday_baking_competition.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping__holiday_baking_competition", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 140800 tokens (140800 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 366.3496813774109, + "token_usage": { + "input_tokens": 1978847, + "output_tokens": 2446, + "total_tokens": 1981293, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 137967 tokens (137967 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092620021422115971434472854)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 321.6748421192169, + "token_usage": { + "input_tokens": 1609481, + "output_tokens": 1812, + "total_tokens": 1611293, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 701.5529799461365, + "token_usage": { + "input_tokens": 1228856, + "output_tokens": 1515, + "total_tokens": 1230371, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 19.334036350250244, + "token_usage": { + "input_tokens": 4383, + "output_tokens": 372, + "total_tokens": 4755, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping__multi_category_budget_analysis.json b/model_results/deepseek-v3-1-terminus/shopping__multi_category_budget_analysis.json new file mode 100644 index 00000000..4d0f2fdc --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping__multi_category_budget_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping__multi_category_budget_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 133235 tokens (133235 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 236.3616054058075, + "token_usage": { + "input_tokens": 1157592, + "output_tokens": 1177, + "total_tokens": 1158769, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 133064 tokens (133064 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092620074915659791460350557)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 266.07610630989075, + "token_usage": { + "input_tokens": 1335356, + "output_tokens": 1261, + "total_tokens": 1336617, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 14.579027891159058, + "token_usage": { + "input_tokens": 4477, + "output_tokens": 266, + "total_tokens": 4743, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 948.7483124732971, + "token_usage": { + "input_tokens": 1330972, + "output_tokens": 1332, + "total_tokens": 1332304, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping__printer_keyboard_search.json b/model_results/deepseek-v3-1-terminus/shopping__printer_keyboard_search.json new file mode 100644 index 00000000..7d433680 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping__printer_keyboard_search.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping__printer_keyboard_search", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 233.01101899147034, + "token_usage": { + "input_tokens": 792882, + "output_tokens": 1899, + "total_tokens": 794781, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 209.66281485557556, + "token_usage": { + "input_tokens": 767609, + "output_tokens": 1392, + "total_tokens": 769001, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 11.628572702407837, + "token_usage": { + "input_tokens": 4220, + "output_tokens": 214, + "total_tokens": 4434, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 11.900094270706177, + "token_usage": { + "input_tokens": 4220, + "output_tokens": 250, + "total_tokens": 4470, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping__running_shoes_purchase.json b/model_results/deepseek-v3-1-terminus/shopping__running_shoes_purchase.json new file mode 100644 index 00000000..1a09c99b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping__running_shoes_purchase.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping__running_shoes_purchase", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 190.89328169822693, + "token_usage": { + "input_tokens": 597536, + "output_tokens": 1830, + "total_tokens": 599366, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 176.13676643371582, + "token_usage": { + "input_tokens": 539729, + "output_tokens": 1411, + "total_tokens": 541140, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 13.315120220184326, + "token_usage": { + "input_tokens": 4192, + "output_tokens": 229, + "total_tokens": 4421, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 681.7983357906342, + "token_usage": { + "input_tokens": 842416, + "output_tokens": 1323, + "total_tokens": 843739, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping_admin__customer_segmentation_setup.json b/model_results/deepseek-v3-1-terminus/shopping_admin__customer_segmentation_setup.json new file mode 100644 index 00000000..0e28bcf2 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping_admin__customer_segmentation_setup.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping_admin__customer_segmentation_setup", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 214.1471290588379, + "token_usage": { + "input_tokens": 643699, + "output_tokens": 1428, + "total_tokens": 645127, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 216.8935251235962, + "token_usage": { + "input_tokens": 696510, + "output_tokens": 1270, + "total_tokens": 697780, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 441.1524147987366, + "token_usage": { + "input_tokens": 722523, + "output_tokens": 1569, + "total_tokens": 724092, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": true, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 129973 exceeds the maximum length 98304. Request id: 021758985374377096fce02f344d1557d64e35a80bc1dfdd945bc (tid: 2025092715010578297845123391216)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 1328.1425185203552, + "token_usage": { + "input_tokens": 1033010, + "output_tokens": 1487, + "total_tokens": 1034497, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping_admin__fitness_promotion_strategy.json b/model_results/deepseek-v3-1-terminus/shopping_admin__fitness_promotion_strategy.json new file mode 100644 index 00000000..0510b961 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping_admin__fitness_promotion_strategy.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping_admin__fitness_promotion_strategy", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 136532 tokens (136532 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 176.36079168319702, + "token_usage": { + "input_tokens": 355450, + "output_tokens": 1175, + "total_tokens": 356625, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 158981 tokens (158981 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092620241782418658114609669)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 167.2538239955902, + "token_usage": { + "input_tokens": 236408, + "output_tokens": 1830, + "total_tokens": 238238, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 134788 exceeds the maximum length 98304. Request id: 021758943547430134ed1c48df78627bed887af9228cd3b472a30 (tid: 2025092703250067708585141224744)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 326.64486384391785, + "token_usage": { + "input_tokens": 280645, + "output_tokens": 762, + "total_tokens": 281407, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 12.434375286102295, + "token_usage": { + "input_tokens": 4693, + "output_tokens": 171, + "total_tokens": 4864, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping_admin__marketing_customer_analysis.json b/model_results/deepseek-v3-1-terminus/shopping_admin__marketing_customer_analysis.json new file mode 100644 index 00000000..9654efb3 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping_admin__marketing_customer_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping_admin__marketing_customer_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 266.61186623573303, + "token_usage": { + "input_tokens": 862730, + "output_tokens": 2359, + "total_tokens": 865089, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 260.6061067581177, + "token_usage": { + "input_tokens": 712796, + "output_tokens": 2180, + "total_tokens": 714976, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 10.612432956695557, + "token_usage": { + "input_tokens": 4918, + "output_tokens": 166, + "total_tokens": 5084, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 861.2593252658844, + "token_usage": { + "input_tokens": 1228487, + "output_tokens": 1485, + "total_tokens": 1229972, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping_admin__ny_expansion_analysis.json b/model_results/deepseek-v3-1-terminus/shopping_admin__ny_expansion_analysis.json new file mode 100644 index 00000000..5367921b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping_admin__ny_expansion_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping_admin__ny_expansion_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 257.78277111053467, + "token_usage": { + "input_tokens": 913824, + "output_tokens": 2517, + "total_tokens": 916341, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 296.32229495048523, + "token_usage": { + "input_tokens": 1125864, + "output_tokens": 2369, + "total_tokens": 1128233, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 472.8414161205292, + "token_usage": { + "input_tokens": 767994, + "output_tokens": 1444, + "total_tokens": 769438, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 526.0267624855042, + "token_usage": { + "input_tokens": 769993, + "output_tokens": 1197, + "total_tokens": 771190, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping_admin__products_sales_analysis.json b/model_results/deepseek-v3-1-terminus/shopping_admin__products_sales_analysis.json new file mode 100644 index 00000000..210668f7 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping_admin__products_sales_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping_admin__products_sales_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 144918 tokens (144918 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 81.78900957107544, + "token_usage": { + "input_tokens": 60608, + "output_tokens": 424, + "total_tokens": 61032, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 144938 tokens (144938 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092620363553240109075526589)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 67.3431282043457, + "token_usage": { + "input_tokens": 60943, + "output_tokens": 455, + "total_tokens": 61398, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 144716 exceeds the maximum length 98304. Request id: 0217589443720922edefd8200aa9cbc258772a53817be83fcfa04 (tid: 2025092703385747924667067103729)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 215.31035828590393, + "token_usage": { + "input_tokens": 69537, + "output_tokens": 474, + "total_tokens": 70011, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 12.990563869476318, + "token_usage": { + "input_tokens": 4541, + "output_tokens": 295, + "total_tokens": 4836, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping_admin__sales_inventory_analysis.json b/model_results/deepseek-v3-1-terminus/shopping_admin__sales_inventory_analysis.json new file mode 100644 index 00000000..28ae6678 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping_admin__sales_inventory_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping_admin__sales_inventory_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 152520 tokens (152520 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 110.81503748893738, + "token_usage": { + "input_tokens": 123514, + "output_tokens": 1171, + "total_tokens": 124685, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 147709 tokens (147709 in the messages, 0 in the completion). Please reduce the length of the messages or completion. (tid: 2025092620384858276404001113485)\",\"type\":\"invalid_request_error\",\"param\":\"\",\"code\":\"invalid_request_error\"}}", + "execution_time": 84.94838762283325, + "token_usage": { + "input_tokens": 89734, + "output_tokens": 616, + "total_tokens": 90350, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 9.45291018486023, + "token_usage": { + "input_tokens": 4690, + "output_tokens": 140, + "total_tokens": 4830, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Input length 139598 exceeds the maximum length 98304. Request id: 021758987346534c6f82c6173ae2988dd3f2bc3d220408fa20c21 (tid: 2025092715343245793410635166774)\",\"type\":\"BadRequest\",\"param\":\"\",\"code\":\"InvalidParameter\"}}", + "execution_time": 383.4228253364563, + "token_usage": { + "input_tokens": 38465, + "output_tokens": 3474, + "total_tokens": 41939, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/shopping_admin__search_filtering_operations.json b/model_results/deepseek-v3-1-terminus/shopping_admin__search_filtering_operations.json new file mode 100644 index 00000000..bb6de81e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/shopping_admin__search_filtering_operations.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "shopping_admin__search_filtering_operations", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 419.3902413845062, + "token_usage": { + "input_tokens": 1609071, + "output_tokens": 4041, + "total_tokens": 1613112, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 422.50532579421997, + "token_usage": { + "input_tokens": 1733315, + "output_tokens": 3994, + "total_tokens": 1737309, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 348.3605172634125, + "token_usage": { + "input_tokens": 299768, + "output_tokens": 5065, + "total_tokens": 304833, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 547.2540693283081, + "token_usage": { + "input_tokens": 558959, + "output_tokens": 5737, + "total_tokens": 564696, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/sports__baseball_player_analysis.json b/model_results/deepseek-v3-1-terminus/sports__baseball_player_analysis.json new file mode 100644 index 00000000..6b30a7ab --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/sports__baseball_player_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "sports__baseball_player_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 542.8017468452454, + "token_usage": { + "input_tokens": 1109408, + "output_tokens": 8989, + "total_tokens": 1118397, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 581.1975631713867, + "token_usage": { + "input_tokens": 964152, + "output_tokens": 8318, + "total_tokens": 972470, + "reasoning_tokens": 0 + }, + "turn_count": 46 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 453.684809923172, + "token_usage": { + "input_tokens": 1024908, + "output_tokens": 5830, + "total_tokens": 1030738, + "reasoning_tokens": 0 + }, + "turn_count": 41 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092700001575026695082677795)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.33480954170227, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/sports__participant_report_optimization.json b/model_results/deepseek-v3-1-terminus/sports__participant_report_optimization.json new file mode 100644 index 00000000..f5a9d38d --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/sports__participant_report_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "sports__participant_report_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 185.02157998085022, + "token_usage": { + "input_tokens": 206543, + "output_tokens": 2498, + "total_tokens": 209041, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 198.7837188243866, + "token_usage": { + "input_tokens": 218552, + "output_tokens": 2606, + "total_tokens": 221158, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 190.93105626106262, + "token_usage": { + "input_tokens": 214355, + "output_tokens": 2742, + "total_tokens": 217097, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092700002415847121674146169)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.22848629951477, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/sports__team_roster_management.json b/model_results/deepseek-v3-1-terminus/sports__team_roster_management.json new file mode 100644 index 00000000..a961f6e7 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/sports__team_roster_management.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "sports__team_roster_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 399.92182302474976, + "token_usage": { + "input_tokens": 709903, + "output_tokens": 4718, + "total_tokens": 714621, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 418.0447723865509, + "token_usage": { + "input_tokens": 851051, + "output_tokens": 4494, + "total_tokens": 855545, + "reasoning_tokens": 0 + }, + "turn_count": 46 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 279.40716457366943, + "token_usage": { + "input_tokens": 542925, + "output_tokens": 3054, + "total_tokens": 545979, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092700003260972873074971164)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.270027160644531, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/standard_operating_procedure__deployment_process_sop.json b/model_results/deepseek-v3-1-terminus/standard_operating_procedure__deployment_process_sop.json new file mode 100644 index 00000000..d22027c8 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/standard_operating_procedure__deployment_process_sop.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "standard_operating_procedure__deployment_process_sop", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 398.45121812820435, + "token_usage": { + "input_tokens": 1193083, + "output_tokens": 3748, + "total_tokens": 1196831, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + "run-2": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 506 (char 505)", + "execution_time": 110.67174291610718, + "token_usage": { + "input_tokens": 162119, + "output_tokens": 2083, + "total_tokens": 164202, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 469.4976861476898, + "token_usage": { + "input_tokens": 2488535, + "output_tokens": 4148, + "total_tokens": 2492683, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 438.0975124835968, + "token_usage": { + "input_tokens": 1193063, + "output_tokens": 4136, + "total_tokens": 1197199, + "reasoning_tokens": 0 + }, + "turn_count": 41 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/standard_operating_procedure__section_organization.json b/model_results/deepseek-v3-1-terminus/standard_operating_procedure__section_organization.json new file mode 100644 index 00000000..3ca03757 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/standard_operating_procedure__section_organization.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "standard_operating_procedure__section_organization", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 357.8434314727783, + "token_usage": { + "input_tokens": 956084, + "output_tokens": 3326, + "total_tokens": 959410, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 207.4619016647339, + "token_usage": { + "input_tokens": 402296, + "output_tokens": 3159, + "total_tokens": 405455, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 420.1193721294403, + "token_usage": { + "input_tokens": 1969206, + "output_tokens": 3855, + "total_tokens": 1973061, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 291.2355492115021, + "token_usage": { + "input_tokens": 675882, + "output_tokens": 2796, + "total_tokens": 678678, + "reasoning_tokens": 0 + }, + "turn_count": 28 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/student_database__duplicate_name.json b/model_results/deepseek-v3-1-terminus/student_database__duplicate_name.json new file mode 100644 index 00000000..ab6823f5 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/student_database__duplicate_name.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "student_database__duplicate_name", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 197.29585361480713, + "token_usage": { + "input_tokens": 66217, + "output_tokens": 3887, + "total_tokens": 70104, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 53.68654775619507, + "token_usage": { + "input_tokens": 22475, + "output_tokens": 786, + "total_tokens": 23261, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 12178 (char 12177)", + "execution_time": 173.56470847129822, + "token_usage": { + "input_tokens": 14501, + "output_tokens": 4301, + "total_tokens": 18802, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 68.06507134437561, + "token_usage": { + "input_tokens": 51854, + "output_tokens": 1184, + "total_tokens": 53038, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/student_database__english_talent.json b/model_results/deepseek-v3-1-terminus/student_database__english_talent.json new file mode 100644 index 00000000..4a5944d0 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/student_database__english_talent.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "student_database__english_talent", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 970.915191411972, + "token_usage": { + "input_tokens": 1131984, + "output_tokens": 18708, + "total_tokens": 1150692, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 883.6563920974731, + "token_usage": { + "input_tokens": 3256993, + "output_tokens": 9641, + "total_tokens": 3266634, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-3": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 12608 (char 12607)", + "execution_time": 189.9031000137329, + "token_usage": { + "input_tokens": 42658, + "output_tokens": 4369, + "total_tokens": 47027, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 186.04001474380493, + "token_usage": { + "input_tokens": 173063, + "output_tokens": 3111, + "total_tokens": 176174, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/student_database__gradebased_score.json b/model_results/deepseek-v3-1-terminus/student_database__gradebased_score.json new file mode 100644 index 00000000..18a00679 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/student_database__gradebased_score.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "student_database__gradebased_score", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 562.4351849555969, + "token_usage": { + "input_tokens": 446093, + "output_tokens": 11828, + "total_tokens": 457921, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": "Unterminated string starting at: line 1 column 12597 (char 12596)", + "execution_time": 204.41193294525146, + "token_usage": { + "input_tokens": 81538, + "output_tokens": 4439, + "total_tokens": 85977, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 707.271048784256, + "token_usage": { + "input_tokens": 406390, + "output_tokens": 12729, + "total_tokens": 419119, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 529.8826360702515, + "token_usage": { + "input_tokens": 283385, + "output_tokens": 12245, + "total_tokens": 295630, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/team_projects__priority_tasks_table.json b/model_results/deepseek-v3-1-terminus/team_projects__priority_tasks_table.json new file mode 100644 index 00000000..5d186658 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/team_projects__priority_tasks_table.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "team_projects__priority_tasks_table", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 130.1109175682068, + "token_usage": { + "input_tokens": 132169, + "output_tokens": 1666, + "total_tokens": 133835, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 72.6975200176239, + "token_usage": { + "input_tokens": 111307, + "output_tokens": 1081, + "total_tokens": 112388, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 123.36963844299316, + "token_usage": { + "input_tokens": 140230, + "output_tokens": 1539, + "total_tokens": 141769, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 131.92403602600098, + "token_usage": { + "input_tokens": 130080, + "output_tokens": 1702, + "total_tokens": 131782, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/team_projects__swap_tasks.json b/model_results/deepseek-v3-1-terminus/team_projects__swap_tasks.json new file mode 100644 index 00000000..14759b8b --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/team_projects__swap_tasks.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "team_projects__swap_tasks", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 221.01931285858154, + "token_usage": { + "input_tokens": 608149, + "output_tokens": 2415, + "total_tokens": 610564, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 214.38651371002197, + "token_usage": { + "input_tokens": 310104, + "output_tokens": 3986, + "total_tokens": 314090, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 290.3823037147522, + "token_usage": { + "input_tokens": 707521, + "output_tokens": 3228, + "total_tokens": 710749, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 256.54919052124023, + "token_usage": { + "input_tokens": 598818, + "output_tokens": 2966, + "total_tokens": 601784, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/threestudio__code_locating.json b/model_results/deepseek-v3-1-terminus/threestudio__code_locating.json new file mode 100644 index 00000000..f4e0f4fb --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/threestudio__code_locating.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "threestudio__code_locating", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 96.89892864227295, + "token_usage": { + "input_tokens": 203505, + "output_tokens": 1032, + "total_tokens": 204537, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 75.37726497650146, + "token_usage": { + "input_tokens": 65240, + "output_tokens": 769, + "total_tokens": 66009, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 79.04541826248169, + "token_usage": { + "input_tokens": 99460, + "output_tokens": 618, + "total_tokens": 100078, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 84.73768305778503, + "token_usage": { + "input_tokens": 81940, + "output_tokens": 1045, + "total_tokens": 82985, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/threestudio__output_analysis.json b/model_results/deepseek-v3-1-terminus/threestudio__output_analysis.json new file mode 100644 index 00000000..26961ae9 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/threestudio__output_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "threestudio__output_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 178.57989811897278, + "token_usage": { + "input_tokens": 343452, + "output_tokens": 1928, + "total_tokens": 345380, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 181.03016757965088, + "token_usage": { + "input_tokens": 337921, + "output_tokens": 2148, + "total_tokens": 340069, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 232.0110559463501, + "token_usage": { + "input_tokens": 259428, + "output_tokens": 1761, + "total_tokens": 261189, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 104.23010182380676, + "token_usage": { + "input_tokens": 120890, + "output_tokens": 1255, + "total_tokens": 122145, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/threestudio__requirements_completion.json b/model_results/deepseek-v3-1-terminus/threestudio__requirements_completion.json new file mode 100644 index 00000000..88aca5c2 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/threestudio__requirements_completion.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "threestudio__requirements_completion", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 117.09667921066284, + "token_usage": { + "input_tokens": 342964, + "output_tokens": 1278, + "total_tokens": 344242, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 112.46994829177856, + "token_usage": { + "input_tokens": 121258, + "output_tokens": 1393, + "total_tokens": 122651, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 45.402591705322266, + "token_usage": { + "input_tokens": 25299, + "output_tokens": 628, + "total_tokens": 25927, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 39.34596085548401, + "token_usage": { + "input_tokens": 25242, + "output_tokens": 553, + "total_tokens": 25795, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/toronto_guide__change_color.json b/model_results/deepseek-v3-1-terminus/toronto_guide__change_color.json new file mode 100644 index 00000000..20bb60f9 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/toronto_guide__change_color.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "toronto_guide__change_color", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 317.946799993515, + "token_usage": { + "input_tokens": 593569, + "output_tokens": 4642, + "total_tokens": 598211, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 134.48727774620056, + "token_usage": { + "input_tokens": 372858, + "output_tokens": 1865, + "total_tokens": 374723, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 394.53380060195923, + "token_usage": { + "input_tokens": 460402, + "output_tokens": 5590, + "total_tokens": 465992, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": "Expecting ',' delimiter: line 1 column 236 (char 235)", + "execution_time": 146.48925971984863, + "token_usage": { + "input_tokens": 225045, + "output_tokens": 1937, + "total_tokens": 226982, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/toronto_guide__weekend_adventure_planner.json b/model_results/deepseek-v3-1-terminus/toronto_guide__weekend_adventure_planner.json new file mode 100644 index 00000000..185fa595 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/toronto_guide__weekend_adventure_planner.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "notion", + "task": "toronto_guide__weekend_adventure_planner", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 265.73278069496155, + "token_usage": { + "input_tokens": 547930, + "output_tokens": 2774, + "total_tokens": 550704, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 120.23621654510498, + "token_usage": { + "input_tokens": 162599, + "output_tokens": 2542, + "total_tokens": 165141, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 163.25672841072083, + "token_usage": { + "input_tokens": 410864, + "output_tokens": 1940, + "total_tokens": 412804, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 171.87180161476135, + "token_usage": { + "input_tokens": 216681, + "output_tokens": 2118, + "total_tokens": 218799, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/vectors__dba_vector_analysis.json b/model_results/deepseek-v3-1-terminus/vectors__dba_vector_analysis.json new file mode 100644 index 00000000..8781d706 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/vectors__dba_vector_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "postgres", + "task": "vectors__dba_vector_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 413.6838562488556, + "token_usage": { + "input_tokens": 463477, + "output_tokens": 5439, + "total_tokens": 468916, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 470.38009667396545, + "token_usage": { + "input_tokens": 459903, + "output_tokens": 7276, + "total_tokens": 467179, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 523.7290132045746, + "token_usage": { + "input_tokens": 691900, + "output_tokens": 7239, + "total_tokens": 699139, + "reasoning_tokens": 0 + }, + "turn_count": 46 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: DeepseekException - {\"error\":{\"message\":\"Incorrect model ID. Please request to view the model page or you do not have permission to use this model deepseek-chat (tid: 2025092700004165173452644853132)\",\"type\":\"Aihubmix_api_error\"}}", + "execution_time": 8.236953020095825, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/votenet__dataset_comparison.json b/model_results/deepseek-v3-1-terminus/votenet__dataset_comparison.json new file mode 100644 index 00000000..2aa39920 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/votenet__dataset_comparison.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "votenet__dataset_comparison", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 242.0093071460724, + "token_usage": { + "input_tokens": 824163, + "output_tokens": 2614, + "total_tokens": 826777, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 120.34643077850342, + "token_usage": { + "input_tokens": 269636, + "output_tokens": 1233, + "total_tokens": 270869, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 148.0965621471405, + "token_usage": { + "input_tokens": 222337, + "output_tokens": 1135, + "total_tokens": 223472, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 130.5047891139984, + "token_usage": { + "input_tokens": 224380, + "output_tokens": 1065, + "total_tokens": 225445, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/votenet__debugging.json b/model_results/deepseek-v3-1-terminus/votenet__debugging.json new file mode 100644 index 00000000..dfa338cb --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/votenet__debugging.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "votenet__debugging", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 232.40335512161255, + "token_usage": { + "input_tokens": 680975, + "output_tokens": 2442, + "total_tokens": 683417, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 117.13639378547668, + "token_usage": { + "input_tokens": 147555, + "output_tokens": 1338, + "total_tokens": 148893, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 223.7998788356781, + "token_usage": { + "input_tokens": 247688, + "output_tokens": 2332, + "total_tokens": 250020, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 268.47444128990173, + "token_usage": { + "input_tokens": 459937, + "output_tokens": 1628, + "total_tokens": 461565, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/votenet__requirements_writing.json b/model_results/deepseek-v3-1-terminus/votenet__requirements_writing.json new file mode 100644 index 00000000..5bcc8061 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/votenet__requirements_writing.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "filesystem", + "task": "votenet__requirements_writing", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 158.32075667381287, + "token_usage": { + "input_tokens": 142084, + "output_tokens": 1816, + "total_tokens": 143900, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 131.65834617614746, + "token_usage": { + "input_tokens": 95521, + "output_tokens": 1617, + "total_tokens": 97138, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 214.7246491909027, + "token_usage": { + "input_tokens": 151789, + "output_tokens": 3352, + "total_tokens": 155141, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 161.85045337677002, + "token_usage": { + "input_tokens": 260578, + "output_tokens": 1482, + "total_tokens": 262060, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/web_search__birth_of_arvinxu.json b/model_results/deepseek-v3-1-terminus/web_search__birth_of_arvinxu.json new file mode 100644 index 00000000..e2c6ac90 --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/web_search__birth_of_arvinxu.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "web_search__birth_of_arvinxu", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 159.4384844303131, + "token_usage": { + "input_tokens": 512478, + "output_tokens": 1521, + "total_tokens": 513999, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 178.98334622383118, + "token_usage": { + "input_tokens": 394513, + "output_tokens": 1210, + "total_tokens": 395723, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 81.99392771720886, + "token_usage": { + "input_tokens": 141518, + "output_tokens": 830, + "total_tokens": 142348, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 139.45248317718506, + "token_usage": { + "input_tokens": 328730, + "output_tokens": 1136, + "total_tokens": 329866, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/deepseek-v3-1-terminus/web_search__r1_arxiv.json b/model_results/deepseek-v3-1-terminus/web_search__r1_arxiv.json new file mode 100644 index 00000000..73a3b43e --- /dev/null +++ b/model_results/deepseek-v3-1-terminus/web_search__r1_arxiv.json @@ -0,0 +1,55 @@ +{ + "model": "deepseek-v3-1-terminus", + "service": "playwright", + "task": "web_search__r1_arxiv", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 136112 tokens (136112 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 140.3222622871399, + "token_usage": { + "input_tokens": 632961, + "output_tokens": 832, + "total_tokens": 633793, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 153474 tokens (153474 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 107.73799419403076, + "token_usage": { + "input_tokens": 206993, + "output_tokens": 445, + "total_tokens": 207438, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 148818 tokens (148818 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 166.31330966949463, + "token_usage": { + "input_tokens": 580182, + "output_tokens": 775, + "total_tokens": 580957, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": "litellm.ContextWindowExceededError: litellm.BadRequestError: ContextWindowExceededError: DeepseekException - {\"error\":{\"message\":\"This model's maximum context length is 131072 tokens. However, you requested 138198 tokens (138198 in the messages, 0 in the completion). Please reduce the length of the messages or completion.\",\"type\":\"invalid_request_error\",\"param\":null,\"code\":\"invalid_request_error\"}}", + "execution_time": 198.00503611564636, + "token_usage": { + "input_tokens": 962489, + "output_tokens": 1062, + "total_tokens": 963551, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/build_your_own_x__find_commit_date.json b/model_results/glm-4-6/build_your_own_x__find_commit_date.json new file mode 100644 index 00000000..91b18fe0 --- /dev/null +++ b/model_results/glm-4-6/build_your_own_x__find_commit_date.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "build_your_own_x__find_commit_date", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 240252 tokens (232175 of text input, 8077 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 533.7350254058838, + "token_usage": { + "input_tokens": 3259116, + "output_tokens": 3047, + "total_tokens": 3262163, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 243.50679802894592, + "token_usage": { + "input_tokens": 1265445, + "output_tokens": 1098, + "total_tokens": 1266543, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 228452 tokens (220375 of text input, 8077 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 184.39719676971436, + "token_usage": { + "input_tokens": 1070156, + "output_tokens": 515, + "total_tokens": 1070671, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 252.76967668533325, + "token_usage": { + "input_tokens": 1769908, + "output_tokens": 1489, + "total_tokens": 1771397, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/build_your_own_x__find_rag_commit.json b/model_results/glm-4-6/build_your_own_x__find_rag_commit.json new file mode 100644 index 00000000..e8186cc3 --- /dev/null +++ b/model_results/glm-4-6/build_your_own_x__find_rag_commit.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "build_your_own_x__find_rag_commit", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 128.51300883293152, + "token_usage": { + "input_tokens": 1472839, + "output_tokens": 1133, + "total_tokens": 1473972, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 240505 tokens (232428 of text input, 8077 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 321.92074847221375, + "token_usage": { + "input_tokens": 1774028, + "output_tokens": 824, + "total_tokens": 1774852, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 206193 tokens (198116 of text input, 8077 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 173.39985013008118, + "token_usage": { + "input_tokens": 932115, + "output_tokens": 839, + "total_tokens": 932954, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 230.04617857933044, + "token_usage": { + "input_tokens": 1667592, + "output_tokens": 2403, + "total_tokens": 1669995, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/chinook__customer_data_migration.json b/model_results/glm-4-6/chinook__customer_data_migration.json new file mode 100644 index 00000000..421266d5 --- /dev/null +++ b/model_results/glm-4-6/chinook__customer_data_migration.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "chinook__customer_data_migration", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 139.31443881988525, + "token_usage": { + "input_tokens": 78384, + "output_tokens": 337, + "total_tokens": 78721, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 150.28711557388306, + "token_usage": { + "input_tokens": 78208, + "output_tokens": 682, + "total_tokens": 78890, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 158.42241668701172, + "token_usage": { + "input_tokens": 78015, + "output_tokens": 646, + "total_tokens": 78661, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 163.1177361011505, + "token_usage": { + "input_tokens": 78451, + "output_tokens": 1880, + "total_tokens": 80331, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/chinook__employee_hierarchy_management.json b/model_results/glm-4-6/chinook__employee_hierarchy_management.json new file mode 100644 index 00000000..413a779d --- /dev/null +++ b/model_results/glm-4-6/chinook__employee_hierarchy_management.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "chinook__employee_hierarchy_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 145.07915711402893, + "token_usage": { + "input_tokens": 179941, + "output_tokens": 3414, + "total_tokens": 183355, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 213.0750117301941, + "token_usage": { + "input_tokens": 205972, + "output_tokens": 3533, + "total_tokens": 209505, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 284.17301630973816, + "token_usage": { + "input_tokens": 233220, + "output_tokens": 3776, + "total_tokens": 236996, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 120.56951808929443, + "token_usage": { + "input_tokens": 297469, + "output_tokens": 3471, + "total_tokens": 300940, + "reasoning_tokens": 0 + }, + "turn_count": 29 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/chinook__sales_and_music_charts.json b/model_results/glm-4-6/chinook__sales_and_music_charts.json new file mode 100644 index 00000000..dec7e8ef --- /dev/null +++ b/model_results/glm-4-6/chinook__sales_and_music_charts.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "chinook__sales_and_music_charts", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 128.95405292510986, + "token_usage": { + "input_tokens": 159822, + "output_tokens": 3325, + "total_tokens": 163147, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 318.0042164325714, + "token_usage": { + "input_tokens": 346957, + "output_tokens": 3706, + "total_tokens": 350663, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 334.34429025650024, + "token_usage": { + "input_tokens": 203671, + "output_tokens": 4180, + "total_tokens": 207851, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 122.83406901359558, + "token_usage": { + "input_tokens": 211858, + "output_tokens": 4209, + "total_tokens": 216067, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/claude-code__automated_changelog_generation.json b/model_results/glm-4-6/claude-code__automated_changelog_generation.json new file mode 100644 index 00000000..e521b547 --- /dev/null +++ b/model_results/glm-4-6/claude-code__automated_changelog_generation.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "claude-code__automated_changelog_generation", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 245.26052379608154, + "token_usage": { + "input_tokens": 344295, + "output_tokens": 6200, + "total_tokens": 350495, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 204.87113118171692, + "token_usage": { + "input_tokens": 368764, + "output_tokens": 6340, + "total_tokens": 375104, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 204.55552554130554, + "token_usage": { + "input_tokens": 372335, + "output_tokens": 7264, + "total_tokens": 379599, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 185.73535060882568, + "token_usage": { + "input_tokens": 335993, + "output_tokens": 6879, + "total_tokens": 342872, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/claude-code__claude_collaboration_analysis.json b/model_results/glm-4-6/claude-code__claude_collaboration_analysis.json new file mode 100644 index 00000000..f40f5e5f --- /dev/null +++ b/model_results/glm-4-6/claude-code__claude_collaboration_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "claude-code__claude_collaboration_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 62.151270627975464, + "token_usage": { + "input_tokens": 207183, + "output_tokens": 1169, + "total_tokens": 208352, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 64.97355198860168, + "token_usage": { + "input_tokens": 207345, + "output_tokens": 1346, + "total_tokens": 208691, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 53.872310638427734, + "token_usage": { + "input_tokens": 262141, + "output_tokens": 1215, + "total_tokens": 263356, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 53.556227684020996, + "token_usage": { + "input_tokens": 208060, + "output_tokens": 1763, + "total_tokens": 209823, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/claude-code__critical_issue_hotfix_workflow.json b/model_results/glm-4-6/claude-code__critical_issue_hotfix_workflow.json new file mode 100644 index 00000000..fc4aa364 --- /dev/null +++ b/model_results/glm-4-6/claude-code__critical_issue_hotfix_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "claude-code__critical_issue_hotfix_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 111.63058614730835, + "token_usage": { + "input_tokens": 163919, + "output_tokens": 2634, + "total_tokens": 166553, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 122.18994617462158, + "token_usage": { + "input_tokens": 164978, + "output_tokens": 2915, + "total_tokens": 167893, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 73.52546429634094, + "token_usage": { + "input_tokens": 161348, + "output_tokens": 1991, + "total_tokens": 163339, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 102.83816957473755, + "token_usage": { + "input_tokens": 305582, + "output_tokens": 4094, + "total_tokens": 309676, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/claude-code__feature_commit_tracking.json b/model_results/glm-4-6/claude-code__feature_commit_tracking.json new file mode 100644 index 00000000..82a3b53c --- /dev/null +++ b/model_results/glm-4-6/claude-code__feature_commit_tracking.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "claude-code__feature_commit_tracking", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 207533 tokens (199456 of text input, 8077 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 180.24797582626343, + "token_usage": { + "input_tokens": 949135, + "output_tokens": 844, + "total_tokens": 949979, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 296.18235182762146, + "token_usage": { + "input_tokens": 1394980, + "output_tokens": 1863, + "total_tokens": 1396843, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 203.18395590782166, + "token_usage": { + "input_tokens": 1098602, + "output_tokens": 1277, + "total_tokens": 1099879, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 161.69209122657776, + "token_usage": { + "input_tokens": 1619263, + "output_tokens": 3772, + "total_tokens": 1623035, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/claude-code__label_color_standardization.json b/model_results/glm-4-6/claude-code__label_color_standardization.json new file mode 100644 index 00000000..509f5455 --- /dev/null +++ b/model_results/glm-4-6/claude-code__label_color_standardization.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "claude-code__label_color_standardization", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 105.3499207496643, + "token_usage": { + "input_tokens": 108063, + "output_tokens": 2462, + "total_tokens": 110525, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 114.19231700897217, + "token_usage": { + "input_tokens": 129718, + "output_tokens": 2504, + "total_tokens": 132222, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 100.34483337402344, + "token_usage": { + "input_tokens": 144454, + "output_tokens": 2860, + "total_tokens": 147314, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 105.05418539047241, + "token_usage": { + "input_tokens": 100743, + "output_tokens": 3466, + "total_tokens": 104209, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/company_in_a_box__employee_onboarding.json b/model_results/glm-4-6/company_in_a_box__employee_onboarding.json new file mode 100644 index 00000000..2ec3b191 --- /dev/null +++ b/model_results/glm-4-6/company_in_a_box__employee_onboarding.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "company_in_a_box__employee_onboarding", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 1406.123925447464, + "token_usage": { + "input_tokens": 6852872, + "output_tokens": 11407, + "total_tokens": 6864279, + "reasoning_tokens": 0 + }, + "turn_count": 79 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 146.4455714225769, + "token_usage": { + "input_tokens": 313058, + "output_tokens": 3139, + "total_tokens": 316197, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 329.0322012901306, + "token_usage": { + "input_tokens": 202914, + "output_tokens": 6891, + "total_tokens": 209805, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 258.61174607276917, + "token_usage": { + "input_tokens": 122120, + "output_tokens": 1653, + "total_tokens": 123773, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/company_in_a_box__goals_restructure.json b/model_results/glm-4-6/company_in_a_box__goals_restructure.json new file mode 100644 index 00000000..d021f394 --- /dev/null +++ b/model_results/glm-4-6/company_in_a_box__goals_restructure.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "company_in_a_box__goals_restructure", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 324.9054024219513, + "token_usage": { + "input_tokens": 2220243, + "output_tokens": 3266, + "total_tokens": 2223509, + "reasoning_tokens": 0 + }, + "turn_count": 48 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 110.3293559551239, + "token_usage": { + "input_tokens": 442067, + "output_tokens": 2007, + "total_tokens": 444074, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 229.26349258422852, + "token_usage": { + "input_tokens": 634486, + "output_tokens": 5096, + "total_tokens": 639582, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 328.98514771461487, + "token_usage": { + "input_tokens": 508801, + "output_tokens": 3659, + "total_tokens": 512460, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/company_in_a_box__quarterly_review_dashboard.json b/model_results/glm-4-6/company_in_a_box__quarterly_review_dashboard.json new file mode 100644 index 00000000..cfb48c56 --- /dev/null +++ b/model_results/glm-4-6/company_in_a_box__quarterly_review_dashboard.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "company_in_a_box__quarterly_review_dashboard", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 806.2172708511353, + "token_usage": { + "input_tokens": 4155605, + "output_tokens": 11648, + "total_tokens": 4167253, + "reasoning_tokens": 0 + }, + "turn_count": 48 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 275.26393938064575, + "token_usage": { + "input_tokens": 1441421, + "output_tokens": 5647, + "total_tokens": 1447068, + "reasoning_tokens": 0 + }, + "turn_count": 43 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 301.31517577171326, + "token_usage": { + "input_tokens": 1073899, + "output_tokens": 6001, + "total_tokens": 1079900, + "reasoning_tokens": 0 + }, + "turn_count": 41 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 746.6911749839783, + "token_usage": { + "input_tokens": 1945983, + "output_tokens": 8320, + "total_tokens": 1954303, + "reasoning_tokens": 0 + }, + "turn_count": 53 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/computer_science_student_dashboard__code_snippets_go.json b/model_results/glm-4-6/computer_science_student_dashboard__code_snippets_go.json new file mode 100644 index 00000000..0df03049 --- /dev/null +++ b/model_results/glm-4-6/computer_science_student_dashboard__code_snippets_go.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "computer_science_student_dashboard__code_snippets_go", + "runs": { + "run-1": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 1152.8275332450867, + "token_usage": { + "input_tokens": 3374753, + "output_tokens": 28840, + "total_tokens": 3403593, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 569.8524651527405, + "token_usage": { + "input_tokens": 2050260, + "output_tokens": 15308, + "total_tokens": 2065568, + "reasoning_tokens": 0 + }, + "turn_count": 56 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 169.74611163139343, + "token_usage": { + "input_tokens": 259855, + "output_tokens": 3940, + "total_tokens": 263795, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": "litellm.APIError: APIError: OpenrouterException - Unable to get json response - Expecting value: line 979 column 1 (char 5379), Original Response:", + "execution_time": 3048.3186750411987, + "token_usage": { + "input_tokens": 605329, + "output_tokens": 10508, + "total_tokens": 615837, + "reasoning_tokens": 0 + }, + "turn_count": 33 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/computer_science_student_dashboard__courses_internships_relation.json b/model_results/glm-4-6/computer_science_student_dashboard__courses_internships_relation.json new file mode 100644 index 00000000..8df09803 --- /dev/null +++ b/model_results/glm-4-6/computer_science_student_dashboard__courses_internships_relation.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "computer_science_student_dashboard__courses_internships_relation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 228.92362999916077, + "token_usage": { + "input_tokens": 339961, + "output_tokens": 4803, + "total_tokens": 344764, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 136.8230242729187, + "token_usage": { + "input_tokens": 465047, + "output_tokens": 2431, + "total_tokens": 467478, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 184.4326102733612, + "token_usage": { + "input_tokens": 373668, + "output_tokens": 5746, + "total_tokens": 379414, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 232.4560890197754, + "token_usage": { + "input_tokens": 381947, + "output_tokens": 4908, + "total_tokens": 386855, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/computer_science_student_dashboard__study_session_tracker.json b/model_results/glm-4-6/computer_science_student_dashboard__study_session_tracker.json new file mode 100644 index 00000000..248d5e83 --- /dev/null +++ b/model_results/glm-4-6/computer_science_student_dashboard__study_session_tracker.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "computer_science_student_dashboard__study_session_tracker", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 83.70585131645203, + "token_usage": { + "input_tokens": 108017, + "output_tokens": 2458, + "total_tokens": 110475, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 85.42530083656311, + "token_usage": { + "input_tokens": 272294, + "output_tokens": 1527, + "total_tokens": 273821, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 111.19457936286926, + "token_usage": { + "input_tokens": 107291, + "output_tokens": 2553, + "total_tokens": 109844, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 95.33345341682434, + "token_usage": { + "input_tokens": 113697, + "output_tokens": 2453, + "total_tokens": 116150, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/desktop__music_report.json b/model_results/glm-4-6/desktop__music_report.json new file mode 100644 index 00000000..78163042 --- /dev/null +++ b/model_results/glm-4-6/desktop__music_report.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "desktop__music_report", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 31.723867654800415, + "token_usage": { + "input_tokens": 24418, + "output_tokens": 961, + "total_tokens": 25379, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 15.10427474975586, + "token_usage": { + "input_tokens": 22831, + "output_tokens": 1097, + "total_tokens": 23928, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 31.723867654800415, + "token_usage": { + "input_tokens": 24418, + "output_tokens": 961, + "total_tokens": 25379, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 15.10427474975586, + "token_usage": { + "input_tokens": 22831, + "output_tokens": 1097, + "total_tokens": 23928, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/desktop__project_management.json b/model_results/glm-4-6/desktop__project_management.json new file mode 100644 index 00000000..0ab24913 --- /dev/null +++ b/model_results/glm-4-6/desktop__project_management.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "desktop__project_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 125.68024682998657, + "token_usage": { + "input_tokens": 442751, + "output_tokens": 7037, + "total_tokens": 449788, + "reasoning_tokens": 0 + }, + "turn_count": 48 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 132.5841419696808, + "token_usage": { + "input_tokens": 603557, + "output_tokens": 7514, + "total_tokens": 611071, + "reasoning_tokens": 0 + }, + "turn_count": 52 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 125.68024682998657, + "token_usage": { + "input_tokens": 442751, + "output_tokens": 7037, + "total_tokens": 449788, + "reasoning_tokens": 0 + }, + "turn_count": 48 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 132.5841419696808, + "token_usage": { + "input_tokens": 603557, + "output_tokens": 7514, + "total_tokens": 611071, + "reasoning_tokens": 0 + }, + "turn_count": 52 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/desktop__timeline_extraction.json b/model_results/glm-4-6/desktop__timeline_extraction.json new file mode 100644 index 00000000..a04e772c --- /dev/null +++ b/model_results/glm-4-6/desktop__timeline_extraction.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "desktop__timeline_extraction", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 57.27190375328064, + "token_usage": { + "input_tokens": 63059, + "output_tokens": 3130, + "total_tokens": 66189, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 26.837627172470093, + "token_usage": { + "input_tokens": 58364, + "output_tokens": 2648, + "total_tokens": 61012, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 57.27190375328064, + "token_usage": { + "input_tokens": 63059, + "output_tokens": 3130, + "total_tokens": 66189, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 26.837627172470093, + "token_usage": { + "input_tokens": 58364, + "output_tokens": 2648, + "total_tokens": 61012, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/desktop_template__budget_computation.json b/model_results/glm-4-6/desktop_template__budget_computation.json new file mode 100644 index 00000000..06cfe609 --- /dev/null +++ b/model_results/glm-4-6/desktop_template__budget_computation.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "desktop_template__budget_computation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 46.019153356552124, + "token_usage": { + "input_tokens": 58174, + "output_tokens": 3070, + "total_tokens": 61244, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 16.283254384994507, + "token_usage": { + "input_tokens": 24004, + "output_tokens": 1281, + "total_tokens": 25285, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 46.019153356552124, + "token_usage": { + "input_tokens": 58174, + "output_tokens": 3070, + "total_tokens": 61244, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 16.283254384994507, + "token_usage": { + "input_tokens": 24004, + "output_tokens": 1281, + "total_tokens": 25285, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/desktop_template__contact_information.json b/model_results/glm-4-6/desktop_template__contact_information.json new file mode 100644 index 00000000..0fd8de05 --- /dev/null +++ b/model_results/glm-4-6/desktop_template__contact_information.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "desktop_template__contact_information", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 44.054227113723755, + "token_usage": { + "input_tokens": 91382, + "output_tokens": 2521, + "total_tokens": 93903, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 33.99298691749573, + "token_usage": { + "input_tokens": 84386, + "output_tokens": 2527, + "total_tokens": 86913, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 44.054227113723755, + "token_usage": { + "input_tokens": 91382, + "output_tokens": 2521, + "total_tokens": 93903, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 33.99298691749573, + "token_usage": { + "input_tokens": 84386, + "output_tokens": 2527, + "total_tokens": 86913, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/desktop_template__file_arrangement.json b/model_results/glm-4-6/desktop_template__file_arrangement.json new file mode 100644 index 00000000..b6c80d22 --- /dev/null +++ b/model_results/glm-4-6/desktop_template__file_arrangement.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "desktop_template__file_arrangement", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 120.52684664726257, + "token_usage": { + "input_tokens": 225019, + "output_tokens": 3962, + "total_tokens": 228981, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 116.76458096504211, + "token_usage": { + "input_tokens": 288160, + "output_tokens": 5002, + "total_tokens": 293162, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 120.52684664726257, + "token_usage": { + "input_tokens": 225019, + "output_tokens": 3962, + "total_tokens": 228981, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 116.76458096504211, + "token_usage": { + "input_tokens": 288160, + "output_tokens": 5002, + "total_tokens": 293162, + "reasoning_tokens": 0 + }, + "turn_count": 37 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/dvdrental__customer_analysis_fix.json b/model_results/glm-4-6/dvdrental__customer_analysis_fix.json new file mode 100644 index 00000000..19b9e2e6 --- /dev/null +++ b/model_results/glm-4-6/dvdrental__customer_analysis_fix.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "dvdrental__customer_analysis_fix", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 206.21510243415833, + "token_usage": { + "input_tokens": 245699, + "output_tokens": 6993, + "total_tokens": 252692, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 477.9997045993805, + "token_usage": { + "input_tokens": 389086, + "output_tokens": 7594, + "total_tokens": 396680, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 435.499480009079, + "token_usage": { + "input_tokens": 321182, + "output_tokens": 7182, + "total_tokens": 328364, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 227.080819606781, + "token_usage": { + "input_tokens": 262025, + "output_tokens": 7421, + "total_tokens": 269446, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/dvdrental__customer_analytics_optimization.json b/model_results/glm-4-6/dvdrental__customer_analytics_optimization.json new file mode 100644 index 00000000..cad4bb3e --- /dev/null +++ b/model_results/glm-4-6/dvdrental__customer_analytics_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "dvdrental__customer_analytics_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 139.83807396888733, + "token_usage": { + "input_tokens": 135409, + "output_tokens": 4690, + "total_tokens": 140099, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": "Invalid control character at: line 1 column 21 (char 20)", + "execution_time": 54.594237089157104, + "token_usage": { + "input_tokens": 28523, + "output_tokens": 1076, + "total_tokens": 29599, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 189.83985376358032, + "token_usage": { + "input_tokens": 229965, + "output_tokens": 3258, + "total_tokens": 233223, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 106.55440258979797, + "token_usage": { + "input_tokens": 119927, + "output_tokens": 3940, + "total_tokens": 123867, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/dvdrental__film_inventory_management.json b/model_results/glm-4-6/dvdrental__film_inventory_management.json new file mode 100644 index 00000000..07188c31 --- /dev/null +++ b/model_results/glm-4-6/dvdrental__film_inventory_management.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "dvdrental__film_inventory_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 172.88974380493164, + "token_usage": { + "input_tokens": 597872, + "output_tokens": 4902, + "total_tokens": 602774, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 311.4297881126404, + "token_usage": { + "input_tokens": 374222, + "output_tokens": 4855, + "total_tokens": 379077, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 247.373144865036, + "token_usage": { + "input_tokens": 467079, + "output_tokens": 5028, + "total_tokens": 472107, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 237.2136902809143, + "token_usage": { + "input_tokens": 584711, + "output_tokens": 5865, + "total_tokens": 590576, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/easyr1__advanced_branch_strategy.json b/model_results/glm-4-6/easyr1__advanced_branch_strategy.json new file mode 100644 index 00000000..96aefd2a --- /dev/null +++ b/model_results/glm-4-6/easyr1__advanced_branch_strategy.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "easyr1__advanced_branch_strategy", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 139.20880770683289, + "token_usage": { + "input_tokens": 205868, + "output_tokens": 2715, + "total_tokens": 208583, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 105.78681325912476, + "token_usage": { + "input_tokens": 223553, + "output_tokens": 1659, + "total_tokens": 225212, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 116.28803777694702, + "token_usage": { + "input_tokens": 342597, + "output_tokens": 1994, + "total_tokens": 344591, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 134.20125770568848, + "token_usage": { + "input_tokens": 286283, + "output_tokens": 3335, + "total_tokens": 289618, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/easyr1__config_parameter_audit.json b/model_results/glm-4-6/easyr1__config_parameter_audit.json new file mode 100644 index 00000000..15ceb800 --- /dev/null +++ b/model_results/glm-4-6/easyr1__config_parameter_audit.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "easyr1__config_parameter_audit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 356.9303925037384, + "token_usage": { + "input_tokens": 1516433, + "output_tokens": 4490, + "total_tokens": 1520923, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 256.49879240989685, + "token_usage": { + "input_tokens": 1576366, + "output_tokens": 1758, + "total_tokens": 1578124, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 249.4454951286316, + "token_usage": { + "input_tokens": 1827788, + "output_tokens": 2158, + "total_tokens": 1829946, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 191.3074803352356, + "token_usage": { + "input_tokens": 1312255, + "output_tokens": 3461, + "total_tokens": 1315716, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/easyr1__performance_regression_investigation.json b/model_results/glm-4-6/easyr1__performance_regression_investigation.json new file mode 100644 index 00000000..854fdca0 --- /dev/null +++ b/model_results/glm-4-6/easyr1__performance_regression_investigation.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "easyr1__performance_regression_investigation", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 181.65913248062134, + "token_usage": { + "input_tokens": 285436, + "output_tokens": 3618, + "total_tokens": 289054, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 198.75509119033813, + "token_usage": { + "input_tokens": 331025, + "output_tokens": 3183, + "total_tokens": 334208, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 96.5533332824707, + "token_usage": { + "input_tokens": 252968, + "output_tokens": 2576, + "total_tokens": 255544, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 178.24584770202637, + "token_usage": { + "input_tokens": 368401, + "output_tokens": 5183, + "total_tokens": 373584, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/easyr1__qwen3_issue_management.json b/model_results/glm-4-6/easyr1__qwen3_issue_management.json new file mode 100644 index 00000000..84e52244 --- /dev/null +++ b/model_results/glm-4-6/easyr1__qwen3_issue_management.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "easyr1__qwen3_issue_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 96.23404240608215, + "token_usage": { + "input_tokens": 228550, + "output_tokens": 2100, + "total_tokens": 230650, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 41.0689959526062, + "token_usage": { + "input_tokens": 90512, + "output_tokens": 681, + "total_tokens": 91193, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 60.37042284011841, + "token_usage": { + "input_tokens": 74504, + "output_tokens": 784, + "total_tokens": 75288, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 40.03966999053955, + "token_usage": { + "input_tokens": 89233, + "output_tokens": 1324, + "total_tokens": 90557, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/employees__employee_demographics_report.json b/model_results/glm-4-6/employees__employee_demographics_report.json new file mode 100644 index 00000000..9d6c039d --- /dev/null +++ b/model_results/glm-4-6/employees__employee_demographics_report.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "employees__employee_demographics_report", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 211.34916234016418, + "token_usage": { + "input_tokens": 177350, + "output_tokens": 4606, + "total_tokens": 181956, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 406.7878062725067, + "token_usage": { + "input_tokens": 187873, + "output_tokens": 4188, + "total_tokens": 192061, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 232.28180265426636, + "token_usage": { + "input_tokens": 205272, + "output_tokens": 4174, + "total_tokens": 209446, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 172.16791248321533, + "token_usage": { + "input_tokens": 284662, + "output_tokens": 4580, + "total_tokens": 289242, + "reasoning_tokens": 0 + }, + "turn_count": 30 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/employees__employee_performance_analysis.json b/model_results/glm-4-6/employees__employee_performance_analysis.json new file mode 100644 index 00000000..5b70d879 --- /dev/null +++ b/model_results/glm-4-6/employees__employee_performance_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "employees__employee_performance_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 184.1734356880188, + "token_usage": { + "input_tokens": 273991, + "output_tokens": 5706, + "total_tokens": 279697, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 223.809636592865, + "token_usage": { + "input_tokens": 140996, + "output_tokens": 3991, + "total_tokens": 144987, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 131.49172067642212, + "token_usage": { + "input_tokens": 141717, + "output_tokens": 4077, + "total_tokens": 145794, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 141.07456827163696, + "token_usage": { + "input_tokens": 192246, + "output_tokens": 4091, + "total_tokens": 196337, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/employees__employee_project_tracking.json b/model_results/glm-4-6/employees__employee_project_tracking.json new file mode 100644 index 00000000..5ce3c246 --- /dev/null +++ b/model_results/glm-4-6/employees__employee_project_tracking.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "employees__employee_project_tracking", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 310.671263217926, + "token_usage": { + "input_tokens": 380970, + "output_tokens": 5956, + "total_tokens": 386926, + "reasoning_tokens": 0 + }, + "turn_count": 41 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 242.3810329437256, + "token_usage": { + "input_tokens": 392049, + "output_tokens": 5494, + "total_tokens": 397543, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + "run-3": { + "success": false, + "error_message": "litellm.APIError: APIError: OpenrouterException - {\"error\":{\"message\":\"Internal Server Error\",\"code\":500}}", + "execution_time": 82.74441766738892, + "token_usage": { + "input_tokens": 92605, + "output_tokens": 1495, + "total_tokens": 94100, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 224.98572301864624, + "token_usage": { + "input_tokens": 433829, + "output_tokens": 6248, + "total_tokens": 440077, + "reasoning_tokens": 0 + }, + "turn_count": 44 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/employees__employee_retention_analysis.json b/model_results/glm-4-6/employees__employee_retention_analysis.json new file mode 100644 index 00000000..ba453480 --- /dev/null +++ b/model_results/glm-4-6/employees__employee_retention_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "employees__employee_retention_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 201.9079568386078, + "token_usage": { + "input_tokens": 159362, + "output_tokens": 6049, + "total_tokens": 165411, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 358.5479097366333, + "token_usage": { + "input_tokens": 181584, + "output_tokens": 5361, + "total_tokens": 186945, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 10.116178035736084, + "token_usage": { + "input_tokens": 7360, + "output_tokens": 200, + "total_tokens": 7560, + "reasoning_tokens": 0 + }, + "turn_count": 3 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 149.96325588226318, + "token_usage": { + "input_tokens": 141993, + "output_tokens": 4819, + "total_tokens": 146812, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/employees__executive_dashboard_automation.json b/model_results/glm-4-6/employees__executive_dashboard_automation.json new file mode 100644 index 00000000..21577c4c --- /dev/null +++ b/model_results/glm-4-6/employees__executive_dashboard_automation.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "employees__executive_dashboard_automation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 182.29265069961548, + "token_usage": { + "input_tokens": 376611, + "output_tokens": 4875, + "total_tokens": 381486, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 380.42989230155945, + "token_usage": { + "input_tokens": 271454, + "output_tokens": 3455, + "total_tokens": 274909, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 233.18688774108887, + "token_usage": { + "input_tokens": 321311, + "output_tokens": 5571, + "total_tokens": 326882, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 194.60288000106812, + "token_usage": { + "input_tokens": 282773, + "output_tokens": 5174, + "total_tokens": 287947, + "reasoning_tokens": 0 + }, + "turn_count": 31 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/employees__management_structure_analysis.json b/model_results/glm-4-6/employees__management_structure_analysis.json new file mode 100644 index 00000000..6136e881 --- /dev/null +++ b/model_results/glm-4-6/employees__management_structure_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "employees__management_structure_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 185.06957983970642, + "token_usage": { + "input_tokens": 259300, + "output_tokens": 5972, + "total_tokens": 265272, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 522.5048370361328, + "token_usage": { + "input_tokens": 320170, + "output_tokens": 5118, + "total_tokens": 325288, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 397.9401066303253, + "token_usage": { + "input_tokens": 344172, + "output_tokens": 5855, + "total_tokens": 350027, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 151.05041193962097, + "token_usage": { + "input_tokens": 212779, + "output_tokens": 4759, + "total_tokens": 217538, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/eval_web__cloudflare_turnstile_challenge.json b/model_results/glm-4-6/eval_web__cloudflare_turnstile_challenge.json new file mode 100644 index 00000000..a1452f65 --- /dev/null +++ b/model_results/glm-4-6/eval_web__cloudflare_turnstile_challenge.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "eval_web__cloudflare_turnstile_challenge", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=z-ai/glm-4.6\n Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers", + "execution_time": 7.055896759033203, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 377.9859049320221, + "token_usage": { + "input_tokens": 688625, + "output_tokens": 9010, + "total_tokens": 697635, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 343.5885910987854, + "token_usage": { + "input_tokens": 665935, + "output_tokens": 9295, + "total_tokens": 675230, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 142.8690106868744, + "token_usage": { + "input_tokens": 169294, + "output_tokens": 4404, + "total_tokens": 173698, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/eval_web__extraction_table.json b/model_results/glm-4-6/eval_web__extraction_table.json new file mode 100644 index 00000000..6a8386d9 --- /dev/null +++ b/model_results/glm-4-6/eval_web__extraction_table.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "eval_web__extraction_table", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 307.0022292137146, + "token_usage": { + "input_tokens": 628481, + "output_tokens": 6534, + "total_tokens": 635015, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 319.55122542381287, + "token_usage": { + "input_tokens": 644006, + "output_tokens": 8224, + "total_tokens": 652230, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 227.08725810050964, + "token_usage": { + "input_tokens": 194261, + "output_tokens": 5557, + "total_tokens": 199818, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 163.77771735191345, + "token_usage": { + "input_tokens": 66785, + "output_tokens": 6079, + "total_tokens": 72864, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/file_context__duplicates_searching.json b/model_results/glm-4-6/file_context__duplicates_searching.json new file mode 100644 index 00000000..4c0163c4 --- /dev/null +++ b/model_results/glm-4-6/file_context__duplicates_searching.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "file_context__duplicates_searching", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 61.74481463432312, + "token_usage": { + "input_tokens": 70967, + "output_tokens": 2367, + "total_tokens": 73334, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 43.0725634098053, + "token_usage": { + "input_tokens": 69579, + "output_tokens": 2440, + "total_tokens": 72019, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 61.74481463432312, + "token_usage": { + "input_tokens": 70967, + "output_tokens": 2367, + "total_tokens": 73334, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 43.0725634098053, + "token_usage": { + "input_tokens": 69579, + "output_tokens": 2440, + "total_tokens": 72019, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/file_context__file_merging.json b/model_results/glm-4-6/file_context__file_merging.json new file mode 100644 index 00000000..b32f68ea --- /dev/null +++ b/model_results/glm-4-6/file_context__file_merging.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "file_context__file_merging", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 26.982781171798706, + "token_usage": { + "input_tokens": 13787, + "output_tokens": 1318, + "total_tokens": 15105, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 13.764727592468262, + "token_usage": { + "input_tokens": 13840, + "output_tokens": 1254, + "total_tokens": 15094, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 26.982781171798706, + "token_usage": { + "input_tokens": 13787, + "output_tokens": 1318, + "total_tokens": 15105, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 13.764727592468262, + "token_usage": { + "input_tokens": 13840, + "output_tokens": 1254, + "total_tokens": 15094, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/file_context__file_splitting.json b/model_results/glm-4-6/file_context__file_splitting.json new file mode 100644 index 00000000..95f55cc1 --- /dev/null +++ b/model_results/glm-4-6/file_context__file_splitting.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "file_context__file_splitting", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 263.3165783882141, + "token_usage": { + "input_tokens": 1184163, + "output_tokens": 11529, + "total_tokens": 1195692, + "reasoning_tokens": 0 + }, + "turn_count": 92 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 113.75364208221436, + "token_usage": { + "input_tokens": 394780, + "output_tokens": 6695, + "total_tokens": 401475, + "reasoning_tokens": 0 + }, + "turn_count": 50 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 263.3165783882141, + "token_usage": { + "input_tokens": 1184163, + "output_tokens": 11529, + "total_tokens": 1195692, + "reasoning_tokens": 0 + }, + "turn_count": 92 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 113.75364208221436, + "token_usage": { + "input_tokens": 394780, + "output_tokens": 6695, + "total_tokens": 401475, + "reasoning_tokens": 0 + }, + "turn_count": 50 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/file_context__pattern_matching.json b/model_results/glm-4-6/file_context__pattern_matching.json new file mode 100644 index 00000000..fc4927bb --- /dev/null +++ b/model_results/glm-4-6/file_context__pattern_matching.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "file_context__pattern_matching", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 108.85073900222778, + "token_usage": { + "input_tokens": 184509, + "output_tokens": 3979, + "total_tokens": 188488, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 34.216456174850464, + "token_usage": { + "input_tokens": 41998, + "output_tokens": 3474, + "total_tokens": 45472, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 108.85073900222778, + "token_usage": { + "input_tokens": 184509, + "output_tokens": 3979, + "total_tokens": 188488, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 34.216456174850464, + "token_usage": { + "input_tokens": 41998, + "output_tokens": 3474, + "total_tokens": 45472, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/file_context__uppercase.json b/model_results/glm-4-6/file_context__uppercase.json new file mode 100644 index 00000000..367ef013 --- /dev/null +++ b/model_results/glm-4-6/file_context__uppercase.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "file_context__uppercase", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 42.0013427734375, + "token_usage": { + "input_tokens": 75119, + "output_tokens": 2687, + "total_tokens": 77806, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 38.24220895767212, + "token_usage": { + "input_tokens": 80457, + "output_tokens": 2638, + "total_tokens": 83095, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 42.0013427734375, + "token_usage": { + "input_tokens": 75119, + "output_tokens": 2687, + "total_tokens": 77806, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 38.24220895767212, + "token_usage": { + "input_tokens": 80457, + "output_tokens": 2638, + "total_tokens": 83095, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/file_property__size_classification.json b/model_results/glm-4-6/file_property__size_classification.json new file mode 100644 index 00000000..7d16f147 --- /dev/null +++ b/model_results/glm-4-6/file_property__size_classification.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "file_property__size_classification", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 67.11552572250366, + "token_usage": { + "input_tokens": 136581, + "output_tokens": 3459, + "total_tokens": 140040, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 29.143349409103394, + "token_usage": { + "input_tokens": 66522, + "output_tokens": 2046, + "total_tokens": 68568, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 67.11552572250366, + "token_usage": { + "input_tokens": 136581, + "output_tokens": 3459, + "total_tokens": 140040, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 29.143349409103394, + "token_usage": { + "input_tokens": 66522, + "output_tokens": 2046, + "total_tokens": 68568, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/file_property__time_classification.json b/model_results/glm-4-6/file_property__time_classification.json new file mode 100644 index 00000000..51854a9c --- /dev/null +++ b/model_results/glm-4-6/file_property__time_classification.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "file_property__time_classification", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 49.26250123977661, + "token_usage": { + "input_tokens": 111501, + "output_tokens": 2696, + "total_tokens": 114197, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 50.258808851242065, + "token_usage": { + "input_tokens": 119795, + "output_tokens": 2766, + "total_tokens": 122561, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 49.26250123977661, + "token_usage": { + "input_tokens": 111501, + "output_tokens": 2696, + "total_tokens": 114197, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 50.258808851242065, + "token_usage": { + "input_tokens": 119795, + "output_tokens": 2766, + "total_tokens": 122561, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/folder_structure__structure_analysis.json b/model_results/glm-4-6/folder_structure__structure_analysis.json new file mode 100644 index 00000000..e19e5dc8 --- /dev/null +++ b/model_results/glm-4-6/folder_structure__structure_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "folder_structure__structure_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 176.78489065170288, + "token_usage": { + "input_tokens": 731419, + "output_tokens": 7317, + "total_tokens": 738736, + "reasoning_tokens": 0 + }, + "turn_count": 57 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 178.56282258033752, + "token_usage": { + "input_tokens": 682892, + "output_tokens": 7135, + "total_tokens": 690027, + "reasoning_tokens": 0 + }, + "turn_count": 55 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 176.78489065170288, + "token_usage": { + "input_tokens": 731419, + "output_tokens": 7317, + "total_tokens": 738736, + "reasoning_tokens": 0 + }, + "turn_count": 57 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 178.56282258033752, + "token_usage": { + "input_tokens": 682892, + "output_tokens": 7135, + "total_tokens": 690027, + "reasoning_tokens": 0 + }, + "turn_count": 55 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/folder_structure__structure_mirror.json b/model_results/glm-4-6/folder_structure__structure_mirror.json new file mode 100644 index 00000000..78dec863 --- /dev/null +++ b/model_results/glm-4-6/folder_structure__structure_mirror.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "folder_structure__structure_mirror", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 153.52127957344055, + "token_usage": { + "input_tokens": 400620, + "output_tokens": 5420, + "total_tokens": 406040, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 109.83824896812439, + "token_usage": { + "input_tokens": 495318, + "output_tokens": 6048, + "total_tokens": 501366, + "reasoning_tokens": 0 + }, + "turn_count": 47 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 153.52127957344055, + "token_usage": { + "input_tokens": 400620, + "output_tokens": 5420, + "total_tokens": 406040, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 109.83824896812439, + "token_usage": { + "input_tokens": 495318, + "output_tokens": 6048, + "total_tokens": 501366, + "reasoning_tokens": 0 + }, + "turn_count": 47 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/harmony__fix_conflict.json b/model_results/glm-4-6/harmony__fix_conflict.json new file mode 100644 index 00000000..3b66e782 --- /dev/null +++ b/model_results/glm-4-6/harmony__fix_conflict.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "harmony__fix_conflict", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 196.27353477478027, + "token_usage": { + "input_tokens": 1336398, + "output_tokens": 3498, + "total_tokens": 1339896, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 231.27164697647095, + "token_usage": { + "input_tokens": 1768764, + "output_tokens": 2841, + "total_tokens": 1771605, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 263.36326789855957, + "token_usage": { + "input_tokens": 2244879, + "output_tokens": 4397, + "total_tokens": 2249276, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 278.4939820766449, + "token_usage": { + "input_tokens": 3126223, + "output_tokens": 6434, + "total_tokens": 3132657, + "reasoning_tokens": 0 + }, + "turn_count": 37 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/harmony__issue_pr_commit_workflow.json b/model_results/glm-4-6/harmony__issue_pr_commit_workflow.json new file mode 100644 index 00000000..c1db77fd --- /dev/null +++ b/model_results/glm-4-6/harmony__issue_pr_commit_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "harmony__issue_pr_commit_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 110.16259336471558, + "token_usage": { + "input_tokens": 367574, + "output_tokens": 2748, + "total_tokens": 370322, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 502.2443449497223, + "token_usage": { + "input_tokens": 1370757, + "output_tokens": 8712, + "total_tokens": 1379469, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 142.06048369407654, + "token_usage": { + "input_tokens": 523834, + "output_tokens": 2476, + "total_tokens": 526310, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 118.41282606124878, + "token_usage": { + "input_tokens": 486304, + "output_tokens": 3438, + "total_tokens": 489742, + "reasoning_tokens": 0 + }, + "turn_count": 16 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/harmony__issue_tagging_pr_closure.json b/model_results/glm-4-6/harmony__issue_tagging_pr_closure.json new file mode 100644 index 00000000..b7b54084 --- /dev/null +++ b/model_results/glm-4-6/harmony__issue_tagging_pr_closure.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "harmony__issue_tagging_pr_closure", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 121.48146224021912, + "token_usage": { + "input_tokens": 196331, + "output_tokens": 2849, + "total_tokens": 199180, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 99.45385217666626, + "token_usage": { + "input_tokens": 160164, + "output_tokens": 2515, + "total_tokens": 162679, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 108.51680445671082, + "token_usage": { + "input_tokens": 159280, + "output_tokens": 2589, + "total_tokens": 161869, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 122.8479917049408, + "token_usage": { + "input_tokens": 343313, + "output_tokens": 4147, + "total_tokens": 347460, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/harmony__multi_branch_commit_aggregation.json b/model_results/glm-4-6/harmony__multi_branch_commit_aggregation.json new file mode 100644 index 00000000..a6d1246f --- /dev/null +++ b/model_results/glm-4-6/harmony__multi_branch_commit_aggregation.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "harmony__multi_branch_commit_aggregation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 336.24945735931396, + "token_usage": { + "input_tokens": 427932, + "output_tokens": 6342, + "total_tokens": 434274, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 121.93927550315857, + "token_usage": { + "input_tokens": 491377, + "output_tokens": 3281, + "total_tokens": 494658, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 173.14394235610962, + "token_usage": { + "input_tokens": 698106, + "output_tokens": 3668, + "total_tokens": 701774, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 161.5775179862976, + "token_usage": { + "input_tokens": 392176, + "output_tokens": 6267, + "total_tokens": 398443, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/harmony__release_management_workflow.json b/model_results/glm-4-6/harmony__release_management_workflow.json new file mode 100644 index 00000000..ad24d7a6 --- /dev/null +++ b/model_results/glm-4-6/harmony__release_management_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "harmony__release_management_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 694.2363014221191, + "token_usage": { + "input_tokens": 2784644, + "output_tokens": 15661, + "total_tokens": 2800305, + "reasoning_tokens": 0 + }, + "turn_count": 55 + }, + "run-2": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 789.9254982471466, + "token_usage": { + "input_tokens": 5021113, + "output_tokens": 16355, + "total_tokens": 5037468, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 483.4850172996521, + "token_usage": { + "input_tokens": 2777271, + "output_tokens": 11096, + "total_tokens": 2788367, + "reasoning_tokens": 0 + }, + "turn_count": 54 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 264.57943320274353, + "token_usage": { + "input_tokens": 939261, + "output_tokens": 10632, + "total_tokens": 949893, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/it_trouble_shooting_hub__asset_retirement_migration.json b/model_results/glm-4-6/it_trouble_shooting_hub__asset_retirement_migration.json new file mode 100644 index 00000000..2dce90e3 --- /dev/null +++ b/model_results/glm-4-6/it_trouble_shooting_hub__asset_retirement_migration.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "it_trouble_shooting_hub__asset_retirement_migration", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 97.5825834274292, + "token_usage": { + "input_tokens": 319635, + "output_tokens": 2674, + "total_tokens": 322309, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 137.63185620307922, + "token_usage": { + "input_tokens": 594531, + "output_tokens": 1876, + "total_tokens": 596407, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 142.25383472442627, + "token_usage": { + "input_tokens": 358252, + "output_tokens": 2753, + "total_tokens": 361005, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 135.81972861289978, + "token_usage": { + "input_tokens": 462541, + "output_tokens": 3062, + "total_tokens": 465603, + "reasoning_tokens": 0 + }, + "turn_count": 16 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/it_trouble_shooting_hub__security_audit_ticket.json b/model_results/glm-4-6/it_trouble_shooting_hub__security_audit_ticket.json new file mode 100644 index 00000000..43f2f21c --- /dev/null +++ b/model_results/glm-4-6/it_trouble_shooting_hub__security_audit_ticket.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "it_trouble_shooting_hub__security_audit_ticket", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 174.7085645198822, + "token_usage": { + "input_tokens": 353844, + "output_tokens": 4284, + "total_tokens": 358128, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 60.417407512664795, + "token_usage": { + "input_tokens": 177608, + "output_tokens": 1083, + "total_tokens": 178691, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 212.88041400909424, + "token_usage": { + "input_tokens": 348894, + "output_tokens": 4805, + "total_tokens": 353699, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 189.17696619033813, + "token_usage": { + "input_tokens": 836349, + "output_tokens": 4582, + "total_tokens": 840931, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/it_trouble_shooting_hub__verification_expired_update.json b/model_results/glm-4-6/it_trouble_shooting_hub__verification_expired_update.json new file mode 100644 index 00000000..792322c0 --- /dev/null +++ b/model_results/glm-4-6/it_trouble_shooting_hub__verification_expired_update.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "it_trouble_shooting_hub__verification_expired_update", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 56.17271971702576, + "token_usage": { + "input_tokens": 203233, + "output_tokens": 1619, + "total_tokens": 204852, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 165.1204538345337, + "token_usage": { + "input_tokens": 461426, + "output_tokens": 4362, + "total_tokens": 465788, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 175.70588207244873, + "token_usage": { + "input_tokens": 304112, + "output_tokens": 3050, + "total_tokens": 307162, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 264.26389479637146, + "token_usage": { + "input_tokens": 749007, + "output_tokens": 5457, + "total_tokens": 754464, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/japan_travel_planner__daily_itinerary_overview.json b/model_results/glm-4-6/japan_travel_planner__daily_itinerary_overview.json new file mode 100644 index 00000000..a2834571 --- /dev/null +++ b/model_results/glm-4-6/japan_travel_planner__daily_itinerary_overview.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "japan_travel_planner__daily_itinerary_overview", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 158.95628833770752, + "token_usage": { + "input_tokens": 316928, + "output_tokens": 3311, + "total_tokens": 320239, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 139.56335735321045, + "token_usage": { + "input_tokens": 244789, + "output_tokens": 3458, + "total_tokens": 248247, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 121.52990245819092, + "token_usage": { + "input_tokens": 146213, + "output_tokens": 3661, + "total_tokens": 149874, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 185.03372359275818, + "token_usage": { + "input_tokens": 159223, + "output_tokens": 3820, + "total_tokens": 163043, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/japan_travel_planner__packing_progress_summary.json b/model_results/glm-4-6/japan_travel_planner__packing_progress_summary.json new file mode 100644 index 00000000..085488ac --- /dev/null +++ b/model_results/glm-4-6/japan_travel_planner__packing_progress_summary.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "japan_travel_planner__packing_progress_summary", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 252.8636691570282, + "token_usage": { + "input_tokens": 1144685, + "output_tokens": 4263, + "total_tokens": 1148948, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 143.458890914917, + "token_usage": { + "input_tokens": 860752, + "output_tokens": 1771, + "total_tokens": 862523, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 224.67261695861816, + "token_usage": { + "input_tokens": 877006, + "output_tokens": 4348, + "total_tokens": 881354, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 240.52384519577026, + "token_usage": { + "input_tokens": 974516, + "output_tokens": 4239, + "total_tokens": 978755, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/japan_travel_planner__remove_osaka_itinerary.json b/model_results/glm-4-6/japan_travel_planner__remove_osaka_itinerary.json new file mode 100644 index 00000000..1fcf25e1 --- /dev/null +++ b/model_results/glm-4-6/japan_travel_planner__remove_osaka_itinerary.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "japan_travel_planner__remove_osaka_itinerary", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 61.98464107513428, + "token_usage": { + "input_tokens": 121115, + "output_tokens": 1487, + "total_tokens": 122602, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 92.15318036079407, + "token_usage": { + "input_tokens": 309254, + "output_tokens": 1103, + "total_tokens": 310357, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 68.0990526676178, + "token_usage": { + "input_tokens": 199959, + "output_tokens": 2119, + "total_tokens": 202078, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 93.30764722824097, + "token_usage": { + "input_tokens": 241781, + "output_tokens": 2180, + "total_tokens": 243961, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/japan_travel_planner__restaurant_expenses_sync.json b/model_results/glm-4-6/japan_travel_planner__restaurant_expenses_sync.json new file mode 100644 index 00000000..abc88112 --- /dev/null +++ b/model_results/glm-4-6/japan_travel_planner__restaurant_expenses_sync.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "japan_travel_planner__restaurant_expenses_sync", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 121.64657688140869, + "token_usage": { + "input_tokens": 134034, + "output_tokens": 2139, + "total_tokens": 136173, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 96.07570600509644, + "token_usage": { + "input_tokens": 416756, + "output_tokens": 1440, + "total_tokens": 418196, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 72.52384734153748, + "token_usage": { + "input_tokens": 273182, + "output_tokens": 2077, + "total_tokens": 275259, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 42.94132041931152, + "token_usage": { + "input_tokens": 118816, + "output_tokens": 1237, + "total_tokens": 120053, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/legal_document__dispute_review.json b/model_results/glm-4-6/legal_document__dispute_review.json new file mode 100644 index 00000000..e0d337e3 --- /dev/null +++ b/model_results/glm-4-6/legal_document__dispute_review.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "legal_document__dispute_review", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 54.23996448516846, + "token_usage": { + "input_tokens": 193401, + "output_tokens": 1785, + "total_tokens": 195186, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 28.6076557636261, + "token_usage": { + "input_tokens": 132418, + "output_tokens": 1321, + "total_tokens": 133739, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 54.23996448516846, + "token_usage": { + "input_tokens": 193401, + "output_tokens": 1785, + "total_tokens": 195186, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 28.6076557636261, + "token_usage": { + "input_tokens": 132418, + "output_tokens": 1321, + "total_tokens": 133739, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/legal_document__individual_comments.json b/model_results/glm-4-6/legal_document__individual_comments.json new file mode 100644 index 00000000..71fe251b --- /dev/null +++ b/model_results/glm-4-6/legal_document__individual_comments.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "legal_document__individual_comments", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 87.5994017124176, + "token_usage": { + "input_tokens": 254346, + "output_tokens": 3925, + "total_tokens": 258271, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 32.986831188201904, + "token_usage": { + "input_tokens": 171583, + "output_tokens": 1930, + "total_tokens": 173513, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 87.5994017124176, + "token_usage": { + "input_tokens": 254346, + "output_tokens": 3925, + "total_tokens": 258271, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 32.986831188201904, + "token_usage": { + "input_tokens": 171583, + "output_tokens": 1930, + "total_tokens": 173513, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/legal_document__solution_tracing.json b/model_results/glm-4-6/legal_document__solution_tracing.json new file mode 100644 index 00000000..16212d6b --- /dev/null +++ b/model_results/glm-4-6/legal_document__solution_tracing.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "legal_document__solution_tracing", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 42.35612201690674, + "token_usage": { + "input_tokens": 209836, + "output_tokens": 1634, + "total_tokens": 211470, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 42.90608882904053, + "token_usage": { + "input_tokens": 210014, + "output_tokens": 1579, + "total_tokens": 211593, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 42.35612201690674, + "token_usage": { + "input_tokens": 209836, + "output_tokens": 1634, + "total_tokens": 211470, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 42.90608882904053, + "token_usage": { + "input_tokens": 210014, + "output_tokens": 1579, + "total_tokens": 211593, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/lego__consistency_enforcement.json b/model_results/glm-4-6/lego__consistency_enforcement.json new file mode 100644 index 00000000..c91160fa --- /dev/null +++ b/model_results/glm-4-6/lego__consistency_enforcement.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "lego__consistency_enforcement", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 338.54951190948486, + "token_usage": { + "input_tokens": 1087510, + "output_tokens": 6107, + "total_tokens": 1093617, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 917.103600025177, + "token_usage": { + "input_tokens": 1636709, + "output_tokens": 7481, + "total_tokens": 1644190, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 519.8713345527649, + "token_usage": { + "input_tokens": 1684524, + "output_tokens": 10347, + "total_tokens": 1694871, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 191.10808682441711, + "token_usage": { + "input_tokens": 332968, + "output_tokens": 6512, + "total_tokens": 339480, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/lego__database_security_policies.json b/model_results/glm-4-6/lego__database_security_policies.json new file mode 100644 index 00000000..d28240c0 --- /dev/null +++ b/model_results/glm-4-6/lego__database_security_policies.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "lego__database_security_policies", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 539499 tokens (538149 of text input, 1350 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 109.89619183540344, + "token_usage": { + "input_tokens": 137413, + "output_tokens": 3005, + "total_tokens": 140418, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 489.69895696640015, + "token_usage": { + "input_tokens": 334309, + "output_tokens": 7466, + "total_tokens": 341775, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 300753 tokens (299403 of text input, 1350 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 157.25187420845032, + "token_usage": { + "input_tokens": 241924, + "output_tokens": 3962, + "total_tokens": 245886, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-4": { + "success": true, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 449023 tokens (447673 of text input, 1350 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 104.91177606582642, + "token_usage": { + "input_tokens": 139041, + "output_tokens": 3023, + "total_tokens": 142064, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/lego__transactional_inventory_transfer.json b/model_results/glm-4-6/lego__transactional_inventory_transfer.json new file mode 100644 index 00000000..457135e7 --- /dev/null +++ b/model_results/glm-4-6/lego__transactional_inventory_transfer.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "lego__transactional_inventory_transfer", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 455.07564425468445, + "token_usage": { + "input_tokens": 765863, + "output_tokens": 13228, + "total_tokens": 779091, + "reasoning_tokens": 0 + }, + "turn_count": 49 + }, + "run-2": { + "success": true, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 219344 tokens (217994 of text input, 1350 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 127.83743572235107, + "token_usage": { + "input_tokens": 55998, + "output_tokens": 3507, + "total_tokens": 59505, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 570.248443365097, + "token_usage": { + "input_tokens": 1274653, + "output_tokens": 13148, + "total_tokens": 1287801, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 437.2577073574066, + "token_usage": { + "input_tokens": 746066, + "output_tokens": 17352, + "total_tokens": 763418, + "reasoning_tokens": 0 + }, + "turn_count": 46 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/mcpmark-cicd__deployment_status_workflow.json b/model_results/glm-4-6/mcpmark-cicd__deployment_status_workflow.json new file mode 100644 index 00000000..56168a2b --- /dev/null +++ b/model_results/glm-4-6/mcpmark-cicd__deployment_status_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "mcpmark-cicd__deployment_status_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 240.17909598350525, + "token_usage": { + "input_tokens": 234759, + "output_tokens": 6379, + "total_tokens": 241138, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 276.39716053009033, + "token_usage": { + "input_tokens": 482961, + "output_tokens": 6822, + "total_tokens": 489783, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": false, + "error_message": "Expecting value: line 1 column 56 (char 55)", + "execution_time": 193.3067033290863, + "token_usage": { + "input_tokens": 272083, + "output_tokens": 5770, + "total_tokens": 277853, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 188.82461166381836, + "token_usage": { + "input_tokens": 280023, + "output_tokens": 7106, + "total_tokens": 287129, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/mcpmark-cicd__issue_management_workflow.json b/model_results/glm-4-6/mcpmark-cicd__issue_management_workflow.json new file mode 100644 index 00000000..3069f156 --- /dev/null +++ b/model_results/glm-4-6/mcpmark-cicd__issue_management_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "mcpmark-cicd__issue_management_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 840.6754078865051, + "token_usage": { + "input_tokens": 2258512, + "output_tokens": 16513, + "total_tokens": 2275025, + "reasoning_tokens": 0 + }, + "turn_count": 61 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 359.1804759502411, + "token_usage": { + "input_tokens": 1088007, + "output_tokens": 9824, + "total_tokens": 1097831, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 506.37666034698486, + "token_usage": { + "input_tokens": 2684040, + "output_tokens": 12887, + "total_tokens": 2696927, + "reasoning_tokens": 0 + }, + "turn_count": 52 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 219096 tokens (211019 of text input, 8077 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 589.6776521205902, + "token_usage": { + "input_tokens": 3772829, + "output_tokens": 8350, + "total_tokens": 3781179, + "reasoning_tokens": 0 + }, + "turn_count": 62 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/mcpmark-cicd__linting_ci_workflow.json b/model_results/glm-4-6/mcpmark-cicd__linting_ci_workflow.json new file mode 100644 index 00000000..d132d03f --- /dev/null +++ b/model_results/glm-4-6/mcpmark-cicd__linting_ci_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "mcpmark-cicd__linting_ci_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 111.29600739479065, + "token_usage": { + "input_tokens": 125234, + "output_tokens": 1998, + "total_tokens": 127232, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 126.32447290420532, + "token_usage": { + "input_tokens": 263845, + "output_tokens": 2706, + "total_tokens": 266551, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 60.407455921173096, + "token_usage": { + "input_tokens": 136487, + "output_tokens": 1918, + "total_tokens": 138405, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 59.993492126464844, + "token_usage": { + "input_tokens": 138206, + "output_tokens": 1862, + "total_tokens": 140068, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/mcpmark-cicd__pr_automation_workflow.json b/model_results/glm-4-6/mcpmark-cicd__pr_automation_workflow.json new file mode 100644 index 00000000..558946ab --- /dev/null +++ b/model_results/glm-4-6/mcpmark-cicd__pr_automation_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "mcpmark-cicd__pr_automation_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 128.08987975120544, + "token_usage": { + "input_tokens": 173599, + "output_tokens": 3824, + "total_tokens": 177423, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 130.76748967170715, + "token_usage": { + "input_tokens": 201366, + "output_tokens": 4673, + "total_tokens": 206039, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 140.8674259185791, + "token_usage": { + "input_tokens": 251204, + "output_tokens": 4737, + "total_tokens": 255941, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 171.66862630844116, + "token_usage": { + "input_tokens": 215372, + "output_tokens": 5907, + "total_tokens": 221279, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/missing-semester__assign_contributor_labels.json b/model_results/glm-4-6/missing-semester__assign_contributor_labels.json new file mode 100644 index 00000000..81cd0d76 --- /dev/null +++ b/model_results/glm-4-6/missing-semester__assign_contributor_labels.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "missing-semester__assign_contributor_labels", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 191.44082021713257, + "token_usage": { + "input_tokens": 1423872, + "output_tokens": 2043, + "total_tokens": 1425915, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 120.27857232093811, + "token_usage": { + "input_tokens": 1040497, + "output_tokens": 1938, + "total_tokens": 1042435, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 130.93763709068298, + "token_usage": { + "input_tokens": 1031150, + "output_tokens": 1733, + "total_tokens": 1032883, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 114.22383570671082, + "token_usage": { + "input_tokens": 974702, + "output_tokens": 1975, + "total_tokens": 976677, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/missing-semester__find_legacy_name.json b/model_results/glm-4-6/missing-semester__find_legacy_name.json new file mode 100644 index 00000000..b34c914a --- /dev/null +++ b/model_results/glm-4-6/missing-semester__find_legacy_name.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "missing-semester__find_legacy_name", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 129.23458671569824, + "token_usage": { + "input_tokens": 641886, + "output_tokens": 884, + "total_tokens": 642770, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 144.26113295555115, + "token_usage": { + "input_tokens": 697641, + "output_tokens": 923, + "total_tokens": 698564, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 232.479074716568, + "token_usage": { + "input_tokens": 1334810, + "output_tokens": 1142, + "total_tokens": 1335952, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 190.98346781730652, + "token_usage": { + "input_tokens": 706977, + "output_tokens": 893, + "total_tokens": 707870, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/missing-semester__find_salient_file.json b/model_results/glm-4-6/missing-semester__find_salient_file.json new file mode 100644 index 00000000..9c0e3f63 --- /dev/null +++ b/model_results/glm-4-6/missing-semester__find_salient_file.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "github", + "task": "missing-semester__find_salient_file", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 647.3342833518982, + "token_usage": { + "input_tokens": 4826411, + "output_tokens": 6408, + "total_tokens": 4832819, + "reasoning_tokens": 0 + }, + "turn_count": 86 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 451.8009171485901, + "token_usage": { + "input_tokens": 4751937, + "output_tokens": 6311, + "total_tokens": 4758248, + "reasoning_tokens": 0 + }, + "turn_count": 85 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 229.7355453968048, + "token_usage": { + "input_tokens": 1697498, + "output_tokens": 5524, + "total_tokens": 1703022, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 402.2834470272064, + "token_usage": { + "input_tokens": 4168805, + "output_tokens": 5807, + "total_tokens": 4174612, + "reasoning_tokens": 0 + }, + "turn_count": 77 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/online_resume__layout_adjustment.json b/model_results/glm-4-6/online_resume__layout_adjustment.json new file mode 100644 index 00000000..333a5207 --- /dev/null +++ b/model_results/glm-4-6/online_resume__layout_adjustment.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "online_resume__layout_adjustment", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 173.49288392066956, + "token_usage": { + "input_tokens": 904879, + "output_tokens": 2968, + "total_tokens": 907847, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + "run-2": { + "success": false, + "error_message": "Expecting value: line 1 column 93 (char 92)", + "execution_time": 122.97894787788391, + "token_usage": { + "input_tokens": 647305, + "output_tokens": 2118, + "total_tokens": 649423, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 321.3583390712738, + "token_usage": { + "input_tokens": 1118278, + "output_tokens": 4761, + "total_tokens": 1123039, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 260.9174680709839, + "token_usage": { + "input_tokens": 1057505, + "output_tokens": 5652, + "total_tokens": 1063157, + "reasoning_tokens": 0 + }, + "turn_count": 34 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/online_resume__projects_section_update.json b/model_results/glm-4-6/online_resume__projects_section_update.json new file mode 100644 index 00000000..f0ff9332 --- /dev/null +++ b/model_results/glm-4-6/online_resume__projects_section_update.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "online_resume__projects_section_update", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 176.80517601966858, + "token_usage": { + "input_tokens": 723756, + "output_tokens": 2837, + "total_tokens": 726593, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 98.39138841629028, + "token_usage": { + "input_tokens": 409236, + "output_tokens": 1843, + "total_tokens": 411079, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 250.32507753372192, + "token_usage": { + "input_tokens": 343554, + "output_tokens": 3333, + "total_tokens": 346887, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 200.62014293670654, + "token_usage": { + "input_tokens": 898213, + "output_tokens": 3919, + "total_tokens": 902132, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/online_resume__skills_development_tracker.json b/model_results/glm-4-6/online_resume__skills_development_tracker.json new file mode 100644 index 00000000..380d742f --- /dev/null +++ b/model_results/glm-4-6/online_resume__skills_development_tracker.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "online_resume__skills_development_tracker", + "runs": { + "run-1": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 1498.7769165039062, + "token_usage": { + "input_tokens": 6542043, + "output_tokens": 24570, + "total_tokens": 6566613, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 218.85236644744873, + "token_usage": { + "input_tokens": 689645, + "output_tokens": 5092, + "total_tokens": 694737, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 386.03705072402954, + "token_usage": { + "input_tokens": 945226, + "output_tokens": 4751, + "total_tokens": 949977, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 180.2046902179718, + "token_usage": { + "input_tokens": 549953, + "output_tokens": 4914, + "total_tokens": 554867, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/online_resume__work_history_addition.json b/model_results/glm-4-6/online_resume__work_history_addition.json new file mode 100644 index 00000000..5be66387 --- /dev/null +++ b/model_results/glm-4-6/online_resume__work_history_addition.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "online_resume__work_history_addition", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 133.6851305961609, + "token_usage": { + "input_tokens": 282638, + "output_tokens": 3660, + "total_tokens": 286298, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 878.5687198638916, + "token_usage": { + "input_tokens": 1089743, + "output_tokens": 31444, + "total_tokens": 1121187, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 325.2312822341919, + "token_usage": { + "input_tokens": 588820, + "output_tokens": 2185, + "total_tokens": 591005, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 262.51826524734497, + "token_usage": { + "input_tokens": 727262, + "output_tokens": 6392, + "total_tokens": 733654, + "reasoning_tokens": 0 + }, + "turn_count": 34 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/papers__author_folders.json b/model_results/glm-4-6/papers__author_folders.json new file mode 100644 index 00000000..b6d9e35e --- /dev/null +++ b/model_results/glm-4-6/papers__author_folders.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "papers__author_folders", + "runs": { + "run-1": { + "success": false, + "error_message": "LLM call failed on step 13: litellm.APIError: APIError: OpenrouterException - Unable to get json response - Expecting value: line 247 column 1 (char 1353), Original Response:", + "execution_time": 191.16957926750183, + "token_usage": { + "input_tokens": 554552, + "output_tokens": 2263, + "total_tokens": 556815, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": "LLM call failed on step 6: litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 420837 tokens (420837 of text input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 30.366920232772827, + "token_usage": { + "input_tokens": 123173, + "output_tokens": 1430, + "total_tokens": 124603, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": "LLM call failed on step 13: litellm.APIError: APIError: OpenrouterException - Unable to get json response - Expecting value: line 247 column 1 (char 1353), Original Response:", + "execution_time": 191.16957926750183, + "token_usage": { + "input_tokens": 554552, + "output_tokens": 2263, + "total_tokens": 556815, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": "LLM call failed on step 6: litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 420837 tokens (420837 of text input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 30.366920232772827, + "token_usage": { + "input_tokens": 123173, + "output_tokens": 1430, + "total_tokens": 124603, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/papers__find_math_paper.json b/model_results/glm-4-6/papers__find_math_paper.json new file mode 100644 index 00000000..6055f697 --- /dev/null +++ b/model_results/glm-4-6/papers__find_math_paper.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "papers__find_math_paper", + "runs": { + "run-1": { + "success": false, + "error_message": "Model produced an invalid response format.", + "execution_time": 3298.3711700439453, + "token_usage": { + "input_tokens": 11920466, + "output_tokens": 57541, + "total_tokens": 11978007, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-2": { + "success": false, + "error_message": "Max iterations (100) reached without a final answer.", + "execution_time": 2546.2523653507233, + "token_usage": { + "input_tokens": 11717689, + "output_tokens": 26675, + "total_tokens": 11744364, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-3": { + "success": false, + "error_message": "Model produced an invalid response format.", + "execution_time": 3298.3711700439453, + "token_usage": { + "input_tokens": 11920466, + "output_tokens": 57541, + "total_tokens": 11978007, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-4": { + "success": false, + "error_message": "Max iterations (100) reached without a final answer.", + "execution_time": 2546.2523653507233, + "token_usage": { + "input_tokens": 11717689, + "output_tokens": 26675, + "total_tokens": 11744364, + "reasoning_tokens": 0 + }, + "turn_count": 100 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/papers__organize_legacy_papers.json b/model_results/glm-4-6/papers__organize_legacy_papers.json new file mode 100644 index 00000000..efac3ef2 --- /dev/null +++ b/model_results/glm-4-6/papers__organize_legacy_papers.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "papers__organize_legacy_papers", + "runs": { + "run-1": { + "success": false, + "error_message": "LLM call failed on step 27: litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 260939 tokens (260939 of text input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 236.41677069664001, + "token_usage": { + "input_tokens": 955019, + "output_tokens": 6264, + "total_tokens": 961283, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 227.79384636878967, + "token_usage": { + "input_tokens": 2429751, + "output_tokens": 5153, + "total_tokens": 2434904, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-3": { + "success": false, + "error_message": "LLM call failed on step 27: litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 260939 tokens (260939 of text input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 236.41677069664001, + "token_usage": { + "input_tokens": 955019, + "output_tokens": 6264, + "total_tokens": 961283, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 227.79384636878967, + "token_usage": { + "input_tokens": 2429751, + "output_tokens": 5153, + "total_tokens": 2434904, + "reasoning_tokens": 0 + }, + "turn_count": 34 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/python_roadmap__expert_level_lessons.json b/model_results/glm-4-6/python_roadmap__expert_level_lessons.json new file mode 100644 index 00000000..13bb82b9 --- /dev/null +++ b/model_results/glm-4-6/python_roadmap__expert_level_lessons.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "python_roadmap__expert_level_lessons", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 260.8379805088043, + "token_usage": { + "input_tokens": 1511897, + "output_tokens": 4258, + "total_tokens": 1516155, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 205.4099669456482, + "token_usage": { + "input_tokens": 932194, + "output_tokens": 3868, + "total_tokens": 936062, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 599.354977607727, + "token_usage": { + "input_tokens": 779376, + "output_tokens": 6302, + "total_tokens": 785678, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 263.7162415981293, + "token_usage": { + "input_tokens": 1523435, + "output_tokens": 6169, + "total_tokens": 1529604, + "reasoning_tokens": 0 + }, + "turn_count": 30 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/python_roadmap__learning_metrics_dashboard.json b/model_results/glm-4-6/python_roadmap__learning_metrics_dashboard.json new file mode 100644 index 00000000..1e43b0f6 --- /dev/null +++ b/model_results/glm-4-6/python_roadmap__learning_metrics_dashboard.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "python_roadmap__learning_metrics_dashboard", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 131.44664645195007, + "token_usage": { + "input_tokens": 292048, + "output_tokens": 2238, + "total_tokens": 294286, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 163.04621577262878, + "token_usage": { + "input_tokens": 935700, + "output_tokens": 3513, + "total_tokens": 939213, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 227.32916402816772, + "token_usage": { + "input_tokens": 358247, + "output_tokens": 3363, + "total_tokens": 361610, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 135.63160467147827, + "token_usage": { + "input_tokens": 281897, + "output_tokens": 3981, + "total_tokens": 285878, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/reddit__ai_data_analyst.json b/model_results/glm-4-6/reddit__ai_data_analyst.json new file mode 100644 index 00000000..8f19b96e --- /dev/null +++ b/model_results/glm-4-6/reddit__ai_data_analyst.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "reddit__ai_data_analyst", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 210.279381275177, + "token_usage": { + "input_tokens": 712821, + "output_tokens": 3192, + "total_tokens": 716013, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 300.31256318092346, + "token_usage": { + "input_tokens": 574886, + "output_tokens": 3217, + "total_tokens": 578103, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 245330 tokens (242712 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 194.1217384338379, + "token_usage": { + "input_tokens": 855805, + "output_tokens": 3326, + "total_tokens": 859131, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 224.93473482131958, + "token_usage": { + "input_tokens": 1120308, + "output_tokens": 4000, + "total_tokens": 1124308, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/reddit__budget_europe_travel.json b/model_results/glm-4-6/reddit__budget_europe_travel.json new file mode 100644 index 00000000..80598f6d --- /dev/null +++ b/model_results/glm-4-6/reddit__budget_europe_travel.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "reddit__budget_europe_travel", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 266.541437625885, + "token_usage": { + "input_tokens": 646523, + "output_tokens": 4384, + "total_tokens": 650907, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 571.070365190506, + "token_usage": { + "input_tokens": 857087, + "output_tokens": 6130, + "total_tokens": 863217, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 178.58940267562866, + "token_usage": { + "input_tokens": 736556, + "output_tokens": 4605, + "total_tokens": 741161, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 251.43928456306458, + "token_usage": { + "input_tokens": 872092, + "output_tokens": 5732, + "total_tokens": 877824, + "reasoning_tokens": 0 + }, + "turn_count": 28 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/reddit__buyitforlife_research.json b/model_results/glm-4-6/reddit__buyitforlife_research.json new file mode 100644 index 00000000..a1de8d19 --- /dev/null +++ b/model_results/glm-4-6/reddit__buyitforlife_research.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "reddit__buyitforlife_research", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 163.68683624267578, + "token_usage": { + "input_tokens": 481128, + "output_tokens": 1675, + "total_tokens": 482803, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 169.3498990535736, + "token_usage": { + "input_tokens": 355013, + "output_tokens": 2337, + "total_tokens": 357350, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 119.77276492118835, + "token_usage": { + "input_tokens": 399690, + "output_tokens": 2167, + "total_tokens": 401857, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 101.64448142051697, + "token_usage": { + "input_tokens": 307110, + "output_tokens": 1842, + "total_tokens": 308952, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/reddit__llm_research_summary.json b/model_results/glm-4-6/reddit__llm_research_summary.json new file mode 100644 index 00000000..f91b9f1d --- /dev/null +++ b/model_results/glm-4-6/reddit__llm_research_summary.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "reddit__llm_research_summary", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 245.22378993034363, + "token_usage": { + "input_tokens": 515961, + "output_tokens": 4574, + "total_tokens": 520535, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 220.0705966949463, + "token_usage": { + "input_tokens": 872219, + "output_tokens": 5022, + "total_tokens": 877241, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 178.38008904457092, + "token_usage": { + "input_tokens": 747936, + "output_tokens": 4806, + "total_tokens": 752742, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 255.52719640731812, + "token_usage": { + "input_tokens": 520757, + "output_tokens": 4694, + "total_tokens": 525451, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/reddit__movie_reviewer_analysis.json b/model_results/glm-4-6/reddit__movie_reviewer_analysis.json new file mode 100644 index 00000000..fbd363d1 --- /dev/null +++ b/model_results/glm-4-6/reddit__movie_reviewer_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "reddit__movie_reviewer_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 996.4635088443756, + "token_usage": { + "input_tokens": 922761, + "output_tokens": 6450, + "total_tokens": 929211, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 292.49811720848083, + "token_usage": { + "input_tokens": 1075048, + "output_tokens": 4264, + "total_tokens": 1079312, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 349.60808777809143, + "token_usage": { + "input_tokens": 1496450, + "output_tokens": 5599, + "total_tokens": 1502049, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 274.4761805534363, + "token_usage": { + "input_tokens": 692459, + "output_tokens": 5781, + "total_tokens": 698240, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/reddit__nba_statistics_analysis.json b/model_results/glm-4-6/reddit__nba_statistics_analysis.json new file mode 100644 index 00000000..2ee5178c --- /dev/null +++ b/model_results/glm-4-6/reddit__nba_statistics_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "reddit__nba_statistics_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 288.4777319431305, + "token_usage": { + "input_tokens": 1011449, + "output_tokens": 1882, + "total_tokens": 1013331, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 180.97325587272644, + "token_usage": { + "input_tokens": 526313, + "output_tokens": 4054, + "total_tokens": 530367, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 212.10700798034668, + "token_usage": { + "input_tokens": 1067637, + "output_tokens": 4827, + "total_tokens": 1072464, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 350.5820713043213, + "token_usage": { + "input_tokens": 998422, + "output_tokens": 4478, + "total_tokens": 1002900, + "reasoning_tokens": 0 + }, + "turn_count": 16 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/reddit__routine_tracker_forum.json b/model_results/glm-4-6/reddit__routine_tracker_forum.json new file mode 100644 index 00000000..bf17c922 --- /dev/null +++ b/model_results/glm-4-6/reddit__routine_tracker_forum.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "reddit__routine_tracker_forum", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 206623 tokens (204005 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 176.59448647499084, + "token_usage": { + "input_tokens": 207392, + "output_tokens": 1506, + "total_tokens": 208898, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 139.69563055038452, + "token_usage": { + "input_tokens": 468396, + "output_tokens": 2132, + "total_tokens": 470528, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 248.80995178222656, + "token_usage": { + "input_tokens": 1241486, + "output_tokens": 2527, + "total_tokens": 1244013, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 206.3200387954712, + "token_usage": { + "input_tokens": 716039, + "output_tokens": 2094, + "total_tokens": 718133, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/security__rls_business_access.json b/model_results/glm-4-6/security__rls_business_access.json new file mode 100644 index 00000000..3478136f --- /dev/null +++ b/model_results/glm-4-6/security__rls_business_access.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "security__rls_business_access", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 299.8071575164795, + "token_usage": { + "input_tokens": 206144, + "output_tokens": 5802, + "total_tokens": 211946, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 434.64122343063354, + "token_usage": { + "input_tokens": 214011, + "output_tokens": 6160, + "total_tokens": 220171, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 188.7235472202301, + "token_usage": { + "input_tokens": 224582, + "output_tokens": 6268, + "total_tokens": 230850, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 218.0106339454651, + "token_usage": { + "input_tokens": 259696, + "output_tokens": 6795, + "total_tokens": 266491, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/security__user_permission_audit.json b/model_results/glm-4-6/security__user_permission_audit.json new file mode 100644 index 00000000..722072ec --- /dev/null +++ b/model_results/glm-4-6/security__user_permission_audit.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "security__user_permission_audit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 178.9361207485199, + "token_usage": { + "input_tokens": 398130, + "output_tokens": 5274, + "total_tokens": 403404, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 226.35493350028992, + "token_usage": { + "input_tokens": 42591, + "output_tokens": 1274, + "total_tokens": 43865, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 190.66862654685974, + "token_usage": { + "input_tokens": 210807, + "output_tokens": 4114, + "total_tokens": 214921, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 140.01296734809875, + "token_usage": { + "input_tokens": 159500, + "output_tokens": 5517, + "total_tokens": 165017, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/self_assessment__faq_column_layout.json b/model_results/glm-4-6/self_assessment__faq_column_layout.json new file mode 100644 index 00000000..f46b5056 --- /dev/null +++ b/model_results/glm-4-6/self_assessment__faq_column_layout.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "self_assessment__faq_column_layout", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 132.00747561454773, + "token_usage": { + "input_tokens": 585424, + "output_tokens": 1822, + "total_tokens": 587246, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 85.53575325012207, + "token_usage": { + "input_tokens": 397865, + "output_tokens": 1783, + "total_tokens": 399648, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 244.30236291885376, + "token_usage": { + "input_tokens": 395464, + "output_tokens": 2801, + "total_tokens": 398265, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 98.54328751564026, + "token_usage": { + "input_tokens": 367330, + "output_tokens": 2850, + "total_tokens": 370180, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/self_assessment__hyperfocus_analysis_report.json b/model_results/glm-4-6/self_assessment__hyperfocus_analysis_report.json new file mode 100644 index 00000000..530ad00a --- /dev/null +++ b/model_results/glm-4-6/self_assessment__hyperfocus_analysis_report.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "self_assessment__hyperfocus_analysis_report", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.APIError: APIError: OpenrouterException - Unable to get json response - Expecting value: line 219 column 1 (char 1199), Original Response:", + "execution_time": 305.67374324798584, + "token_usage": { + "input_tokens": 259608, + "output_tokens": 3642, + "total_tokens": 263250, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": "Expecting value: line 1 column 59 (char 58)", + "execution_time": 86.78921341896057, + "token_usage": { + "input_tokens": 157733, + "output_tokens": 2203, + "total_tokens": 159936, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 343.852254152298, + "token_usage": { + "input_tokens": 408216, + "output_tokens": 4806, + "total_tokens": 413022, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 162.0008807182312, + "token_usage": { + "input_tokens": 566951, + "output_tokens": 5369, + "total_tokens": 572320, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/self_assessment__numbered_list_emojis.json b/model_results/glm-4-6/self_assessment__numbered_list_emojis.json new file mode 100644 index 00000000..59968930 --- /dev/null +++ b/model_results/glm-4-6/self_assessment__numbered_list_emojis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "self_assessment__numbered_list_emojis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 31.017234086990356, + "token_usage": { + "input_tokens": 77635, + "output_tokens": 287, + "total_tokens": 77922, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": "Expecting value: line 1 column 59 (char 58)", + "execution_time": 25.62408208847046, + "token_usage": { + "input_tokens": 77460, + "output_tokens": 222, + "total_tokens": 77682, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 511.9784278869629, + "token_usage": { + "input_tokens": 1066005, + "output_tokens": 4877, + "total_tokens": 1070882, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 158.38496708869934, + "token_usage": { + "input_tokens": 1117600, + "output_tokens": 4180, + "total_tokens": 1121780, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping__advanced_product_analysis.json b/model_results/glm-4-6/shopping__advanced_product_analysis.json new file mode 100644 index 00000000..ed1b03a9 --- /dev/null +++ b/model_results/glm-4-6/shopping__advanced_product_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping__advanced_product_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 365.90656566619873, + "token_usage": { + "input_tokens": 1084750, + "output_tokens": 3353, + "total_tokens": 1088103, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 288.08566522598267, + "token_usage": { + "input_tokens": 965974, + "output_tokens": 3279, + "total_tokens": 969253, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 153.26812481880188, + "token_usage": { + "input_tokens": 958162, + "output_tokens": 3112, + "total_tokens": 961274, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 271.8477392196655, + "token_usage": { + "input_tokens": 1081798, + "output_tokens": 2438, + "total_tokens": 1084236, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping__gaming_accessories_analysis.json b/model_results/glm-4-6/shopping__gaming_accessories_analysis.json new file mode 100644 index 00000000..93291547 --- /dev/null +++ b/model_results/glm-4-6/shopping__gaming_accessories_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping__gaming_accessories_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 429.2561628818512, + "token_usage": { + "input_tokens": 2495652, + "output_tokens": 3591, + "total_tokens": 2499243, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 319.4610013961792, + "token_usage": { + "input_tokens": 1938428, + "output_tokens": 4379, + "total_tokens": 1942807, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 280.0683240890503, + "token_usage": { + "input_tokens": 1839114, + "output_tokens": 5494, + "total_tokens": 1844608, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 473.11012864112854, + "token_usage": { + "input_tokens": 2664906, + "output_tokens": 5481, + "total_tokens": 2670387, + "reasoning_tokens": 0 + }, + "turn_count": 33 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping__health_routine_optimization.json b/model_results/glm-4-6/shopping__health_routine_optimization.json new file mode 100644 index 00000000..adc0d805 --- /dev/null +++ b/model_results/glm-4-6/shopping__health_routine_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping__health_routine_optimization", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 513.5799055099487, + "token_usage": { + "input_tokens": 1541957, + "output_tokens": 5384, + "total_tokens": 1547341, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 417.6526165008545, + "token_usage": { + "input_tokens": 1817885, + "output_tokens": 5266, + "total_tokens": 1823151, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 315.5167520046234, + "token_usage": { + "input_tokens": 1616482, + "output_tokens": 5642, + "total_tokens": 1622124, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 452.0021958351135, + "token_usage": { + "input_tokens": 1405861, + "output_tokens": 4614, + "total_tokens": 1410475, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping__holiday_baking_competition.json b/model_results/glm-4-6/shopping__holiday_baking_competition.json new file mode 100644 index 00000000..cbe25430 --- /dev/null +++ b/model_results/glm-4-6/shopping__holiday_baking_competition.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping__holiday_baking_competition", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 830.770646572113, + "token_usage": { + "input_tokens": 2290402, + "output_tokens": 6994, + "total_tokens": 2297396, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 282.2895929813385, + "token_usage": { + "input_tokens": 1602840, + "output_tokens": 5305, + "total_tokens": 1608145, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 205784 tokens (203166 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 456.966237783432, + "token_usage": { + "input_tokens": 2742848, + "output_tokens": 3090, + "total_tokens": 2745938, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-4": { + "success": false, + "error_message": "'Exception' object has no attribute 'request'", + "execution_time": 647.4424140453339, + "token_usage": { + "input_tokens": 1653402, + "output_tokens": 3781, + "total_tokens": 1657183, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping__multi_category_budget_analysis.json b/model_results/glm-4-6/shopping__multi_category_budget_analysis.json new file mode 100644 index 00000000..15b13fc1 --- /dev/null +++ b/model_results/glm-4-6/shopping__multi_category_budget_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping__multi_category_budget_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 793.4901113510132, + "token_usage": { + "input_tokens": 2580076, + "output_tokens": 5897, + "total_tokens": 2585973, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 393.21907353401184, + "token_usage": { + "input_tokens": 2040006, + "output_tokens": 5397, + "total_tokens": 2045403, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 429.7153789997101, + "token_usage": { + "input_tokens": 2493330, + "output_tokens": 5606, + "total_tokens": 2498936, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 968.8836879730225, + "token_usage": { + "input_tokens": 1507110, + "output_tokens": 3525, + "total_tokens": 1510635, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping__printer_keyboard_search.json b/model_results/glm-4-6/shopping__printer_keyboard_search.json new file mode 100644 index 00000000..7e3192ee --- /dev/null +++ b/model_results/glm-4-6/shopping__printer_keyboard_search.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping__printer_keyboard_search", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 547.8928933143616, + "token_usage": { + "input_tokens": 1286864, + "output_tokens": 5786, + "total_tokens": 1292650, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 110.55178308486938, + "token_usage": { + "input_tokens": 384916, + "output_tokens": 2515, + "total_tokens": 387431, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 259.6538293361664, + "token_usage": { + "input_tokens": 1065550, + "output_tokens": 3600, + "total_tokens": 1069150, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 133.53703093528748, + "token_usage": { + "input_tokens": 34888, + "output_tokens": 463, + "total_tokens": 35351, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping__running_shoes_purchase.json b/model_results/glm-4-6/shopping__running_shoes_purchase.json new file mode 100644 index 00000000..b9399818 --- /dev/null +++ b/model_results/glm-4-6/shopping__running_shoes_purchase.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping__running_shoes_purchase", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 278.5891230106354, + "token_usage": { + "input_tokens": 358437, + "output_tokens": 3075, + "total_tokens": 361512, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 136.38987636566162, + "token_usage": { + "input_tokens": 531715, + "output_tokens": 2361, + "total_tokens": 534076, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 139.0381636619568, + "token_usage": { + "input_tokens": 344980, + "output_tokens": 2858, + "total_tokens": 347838, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 122.1661901473999, + "token_usage": { + "input_tokens": 72262, + "output_tokens": 465, + "total_tokens": 72727, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping_admin__customer_segmentation_setup.json b/model_results/glm-4-6/shopping_admin__customer_segmentation_setup.json new file mode 100644 index 00000000..6ac6d7ce --- /dev/null +++ b/model_results/glm-4-6/shopping_admin__customer_segmentation_setup.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping_admin__customer_segmentation_setup", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 331.5776889324188, + "token_usage": { + "input_tokens": 672410, + "output_tokens": 3064, + "total_tokens": 675474, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 171.1226522922516, + "token_usage": { + "input_tokens": 413752, + "output_tokens": 2902, + "total_tokens": 416654, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 167.30318188667297, + "token_usage": { + "input_tokens": 586153, + "output_tokens": 2781, + "total_tokens": 588934, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 213.3192458152771, + "token_usage": { + "input_tokens": 594703, + "output_tokens": 2829, + "total_tokens": 597532, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping_admin__fitness_promotion_strategy.json b/model_results/glm-4-6/shopping_admin__fitness_promotion_strategy.json new file mode 100644 index 00000000..4154b6bd --- /dev/null +++ b/model_results/glm-4-6/shopping_admin__fitness_promotion_strategy.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping_admin__fitness_promotion_strategy", + "runs": { + "run-1": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 254068 tokens (251450 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 340.52450609207153, + "token_usage": { + "input_tokens": 1426364, + "output_tokens": 2279, + "total_tokens": 1428643, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 228857 tokens (226239 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 173.18343925476074, + "token_usage": { + "input_tokens": 736110, + "output_tokens": 2123, + "total_tokens": 738233, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 210841 tokens (208223 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 198.40538907051086, + "token_usage": { + "input_tokens": 866853, + "output_tokens": 2291, + "total_tokens": 869144, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 194.69450616836548, + "token_usage": { + "input_tokens": 221583, + "output_tokens": 1509, + "total_tokens": 223092, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping_admin__marketing_customer_analysis.json b/model_results/glm-4-6/shopping_admin__marketing_customer_analysis.json new file mode 100644 index 00000000..7f01629c --- /dev/null +++ b/model_results/glm-4-6/shopping_admin__marketing_customer_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping_admin__marketing_customer_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 417.06060910224915, + "token_usage": { + "input_tokens": 1124421, + "output_tokens": 4973, + "total_tokens": 1129394, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 216.70312690734863, + "token_usage": { + "input_tokens": 861370, + "output_tokens": 4513, + "total_tokens": 865883, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 272.5494763851166, + "token_usage": { + "input_tokens": 1161034, + "output_tokens": 5258, + "total_tokens": 1166292, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 325.47608065605164, + "token_usage": { + "input_tokens": 518723, + "output_tokens": 2551, + "total_tokens": 521274, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping_admin__ny_expansion_analysis.json b/model_results/glm-4-6/shopping_admin__ny_expansion_analysis.json new file mode 100644 index 00000000..ed599eb7 --- /dev/null +++ b/model_results/glm-4-6/shopping_admin__ny_expansion_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping_admin__ny_expansion_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 238.30118131637573, + "token_usage": { + "input_tokens": 421551, + "output_tokens": 2833, + "total_tokens": 424384, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 117.67400908470154, + "token_usage": { + "input_tokens": 448270, + "output_tokens": 2810, + "total_tokens": 451080, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 163.60476279258728, + "token_usage": { + "input_tokens": 370831, + "output_tokens": 3040, + "total_tokens": 373871, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 18.722394466400146, + "token_usage": { + "input_tokens": 10626, + "output_tokens": 122, + "total_tokens": 10748, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping_admin__products_sales_analysis.json b/model_results/glm-4-6/shopping_admin__products_sales_analysis.json new file mode 100644 index 00000000..648c5f55 --- /dev/null +++ b/model_results/glm-4-6/shopping_admin__products_sales_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping_admin__products_sales_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 206.77240800857544, + "token_usage": { + "input_tokens": 176306, + "output_tokens": 1450, + "total_tokens": 177756, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 228520 tokens (225902 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 81.09342336654663, + "token_usage": { + "input_tokens": 176232, + "output_tokens": 1359, + "total_tokens": 177591, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 78.4146716594696, + "token_usage": { + "input_tokens": 191188, + "output_tokens": 1326, + "total_tokens": 192514, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 24.652905702590942, + "token_usage": { + "input_tokens": 2791, + "output_tokens": 0, + "total_tokens": 2791, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping_admin__sales_inventory_analysis.json b/model_results/glm-4-6/shopping_admin__sales_inventory_analysis.json new file mode 100644 index 00000000..0b096f3c --- /dev/null +++ b/model_results/glm-4-6/shopping_admin__sales_inventory_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping_admin__sales_inventory_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 210.5877914428711, + "token_usage": { + "input_tokens": 568493, + "output_tokens": 1802, + "total_tokens": 570295, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 230164 tokens (227546 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 109.1310703754425, + "token_usage": { + "input_tokens": 221445, + "output_tokens": 2066, + "total_tokens": 223511, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 315.78510427474976, + "token_usage": { + "input_tokens": 1092282, + "output_tokens": 3167, + "total_tokens": 1095449, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 78.96198010444641, + "token_usage": { + "input_tokens": 11213, + "output_tokens": 282, + "total_tokens": 11495, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/shopping_admin__search_filtering_operations.json b/model_results/glm-4-6/shopping_admin__search_filtering_operations.json new file mode 100644 index 00000000..49d8bade --- /dev/null +++ b/model_results/glm-4-6/shopping_admin__search_filtering_operations.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "shopping_admin__search_filtering_operations", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 498.4454791545868, + "token_usage": { + "input_tokens": 1338290, + "output_tokens": 6084, + "total_tokens": 1344374, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 249.90884447097778, + "token_usage": { + "input_tokens": 1010859, + "output_tokens": 6047, + "total_tokens": 1016906, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 309.37129759788513, + "token_usage": { + "input_tokens": 1228578, + "output_tokens": 5963, + "total_tokens": 1234541, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 7.914529085159302, + "token_usage": { + "input_tokens": 6947, + "output_tokens": 170, + "total_tokens": 7117, + "reasoning_tokens": 0 + }, + "turn_count": 2 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/sports__baseball_player_analysis.json b/model_results/glm-4-6/sports__baseball_player_analysis.json new file mode 100644 index 00000000..39599e1d --- /dev/null +++ b/model_results/glm-4-6/sports__baseball_player_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "sports__baseball_player_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 384.7446782588959, + "token_usage": { + "input_tokens": 835216, + "output_tokens": 9027, + "total_tokens": 844243, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 640.1871137619019, + "token_usage": { + "input_tokens": 701876, + "output_tokens": 8639, + "total_tokens": 710515, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 318.6987292766571, + "token_usage": { + "input_tokens": 922846, + "output_tokens": 7878, + "total_tokens": 930724, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 373.0751893520355, + "token_usage": { + "input_tokens": 1093749, + "output_tokens": 10374, + "total_tokens": 1104123, + "reasoning_tokens": 0 + }, + "turn_count": 44 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/sports__participant_report_optimization.json b/model_results/glm-4-6/sports__participant_report_optimization.json new file mode 100644 index 00000000..9eb2bbf5 --- /dev/null +++ b/model_results/glm-4-6/sports__participant_report_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "sports__participant_report_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 174.52094841003418, + "token_usage": { + "input_tokens": 183212, + "output_tokens": 3138, + "total_tokens": 186350, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 210.1871988773346, + "token_usage": { + "input_tokens": 219995, + "output_tokens": 2557, + "total_tokens": 222552, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 158.8601644039154, + "token_usage": { + "input_tokens": 260501, + "output_tokens": 4276, + "total_tokens": 264777, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 193.19554924964905, + "token_usage": { + "input_tokens": 256325, + "output_tokens": 4033, + "total_tokens": 260358, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/sports__team_roster_management.json b/model_results/glm-4-6/sports__team_roster_management.json new file mode 100644 index 00000000..7a930e35 --- /dev/null +++ b/model_results/glm-4-6/sports__team_roster_management.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "sports__team_roster_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 427.552960395813, + "token_usage": { + "input_tokens": 486967, + "output_tokens": 5060, + "total_tokens": 492027, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 384.1921184062958, + "token_usage": { + "input_tokens": 523628, + "output_tokens": 5379, + "total_tokens": 529007, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 196.74167203903198, + "token_usage": { + "input_tokens": 581351, + "output_tokens": 5199, + "total_tokens": 586550, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 260.12403082847595, + "token_usage": { + "input_tokens": 679800, + "output_tokens": 6210, + "total_tokens": 686010, + "reasoning_tokens": 0 + }, + "turn_count": 35 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/standard_operating_procedure__deployment_process_sop.json b/model_results/glm-4-6/standard_operating_procedure__deployment_process_sop.json new file mode 100644 index 00000000..2d9314be --- /dev/null +++ b/model_results/glm-4-6/standard_operating_procedure__deployment_process_sop.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "standard_operating_procedure__deployment_process_sop", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 477.6065456867218, + "token_usage": { + "input_tokens": 2561464, + "output_tokens": 5155, + "total_tokens": 2566619, + "reasoning_tokens": 0 + }, + "turn_count": 59 + }, + "run-2": { + "success": false, + "error_message": "Expecting value: line 1 column 59 (char 58)", + "execution_time": 31.954814195632935, + "token_usage": { + "input_tokens": 141943, + "output_tokens": 568, + "total_tokens": 142511, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1501.3019998073578, + "token_usage": { + "input_tokens": 4945010, + "output_tokens": 11586, + "total_tokens": 4956596, + "reasoning_tokens": 0 + }, + "turn_count": 82 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 502.6164765357971, + "token_usage": { + "input_tokens": 3365261, + "output_tokens": 6874, + "total_tokens": 3372135, + "reasoning_tokens": 0 + }, + "turn_count": 58 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/standard_operating_procedure__section_organization.json b/model_results/glm-4-6/standard_operating_procedure__section_organization.json new file mode 100644 index 00000000..6e187fc0 --- /dev/null +++ b/model_results/glm-4-6/standard_operating_procedure__section_organization.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "standard_operating_procedure__section_organization", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 162.294846534729, + "token_usage": { + "input_tokens": 554015, + "output_tokens": 3696, + "total_tokens": 557711, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 360.9465434551239, + "token_usage": { + "input_tokens": 1365664, + "output_tokens": 6453, + "total_tokens": 1372117, + "reasoning_tokens": 0 + }, + "turn_count": 45 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 865.7514960765839, + "token_usage": { + "input_tokens": 782817, + "output_tokens": 7515, + "total_tokens": 790332, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 548.4918584823608, + "token_usage": { + "input_tokens": 2457548, + "output_tokens": 7775, + "total_tokens": 2465323, + "reasoning_tokens": 0 + }, + "turn_count": 49 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/student_database__duplicate_name.json b/model_results/glm-4-6/student_database__duplicate_name.json new file mode 100644 index 00000000..91626db3 --- /dev/null +++ b/model_results/glm-4-6/student_database__duplicate_name.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "student_database__duplicate_name", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 104.50471830368042, + "token_usage": { + "input_tokens": 347474, + "output_tokens": 8219, + "total_tokens": 355693, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 36.38350868225098, + "token_usage": { + "input_tokens": 14319, + "output_tokens": 2589, + "total_tokens": 16908, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 104.50471830368042, + "token_usage": { + "input_tokens": 347474, + "output_tokens": 8219, + "total_tokens": 355693, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 36.38350868225098, + "token_usage": { + "input_tokens": 14319, + "output_tokens": 2589, + "total_tokens": 16908, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/student_database__english_talent.json b/model_results/glm-4-6/student_database__english_talent.json new file mode 100644 index 00000000..1a829b32 --- /dev/null +++ b/model_results/glm-4-6/student_database__english_talent.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "student_database__english_talent", + "runs": { + "run-1": { + "success": false, + "error_message": "LLM call failed on step 22: litellm.APIError: APIError: OpenrouterException - Unable to get json response - Expecting value: line 299 column 1 (char 1639), Original Response:", + "execution_time": 334.3310852050781, + "token_usage": { + "input_tokens": 806248, + "output_tokens": 18446, + "total_tokens": 824694, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 268.6572587490082, + "token_usage": { + "input_tokens": 1569649, + "output_tokens": 14019, + "total_tokens": 1583668, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + "run-3": { + "success": false, + "error_message": "LLM call failed on step 22: litellm.APIError: APIError: OpenrouterException - Unable to get json response - Expecting value: line 299 column 1 (char 1639), Original Response:", + "execution_time": 334.3310852050781, + "token_usage": { + "input_tokens": 806248, + "output_tokens": 18446, + "total_tokens": 824694, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 268.6572587490082, + "token_usage": { + "input_tokens": 1569649, + "output_tokens": 14019, + "total_tokens": 1583668, + "reasoning_tokens": 0 + }, + "turn_count": 39 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/student_database__gradebased_score.json b/model_results/glm-4-6/student_database__gradebased_score.json new file mode 100644 index 00000000..364c3fd0 --- /dev/null +++ b/model_results/glm-4-6/student_database__gradebased_score.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "student_database__gradebased_score", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 111.69829726219177, + "token_usage": { + "input_tokens": 213718, + "output_tokens": 10884, + "total_tokens": 224602, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 119.51101922988892, + "token_usage": { + "input_tokens": 213412, + "output_tokens": 11001, + "total_tokens": 224413, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 111.69829726219177, + "token_usage": { + "input_tokens": 213718, + "output_tokens": 10884, + "total_tokens": 224602, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 119.51101922988892, + "token_usage": { + "input_tokens": 213412, + "output_tokens": 11001, + "total_tokens": 224413, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/team_projects__priority_tasks_table.json b/model_results/glm-4-6/team_projects__priority_tasks_table.json new file mode 100644 index 00000000..bbf1172f --- /dev/null +++ b/model_results/glm-4-6/team_projects__priority_tasks_table.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "team_projects__priority_tasks_table", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 55.677526235580444, + "token_usage": { + "input_tokens": 90815, + "output_tokens": 1224, + "total_tokens": 92039, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 86.4115207195282, + "token_usage": { + "input_tokens": 123527, + "output_tokens": 2071, + "total_tokens": 125598, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 188.60025334358215, + "token_usage": { + "input_tokens": 151045, + "output_tokens": 437, + "total_tokens": 151482, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 195.64994049072266, + "token_usage": { + "input_tokens": 296398, + "output_tokens": 4884, + "total_tokens": 301282, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/team_projects__swap_tasks.json b/model_results/glm-4-6/team_projects__swap_tasks.json new file mode 100644 index 00000000..b651d2a0 --- /dev/null +++ b/model_results/glm-4-6/team_projects__swap_tasks.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "team_projects__swap_tasks", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 91.3856053352356, + "token_usage": { + "input_tokens": 492084, + "output_tokens": 1762, + "total_tokens": 493846, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 254.1742558479309, + "token_usage": { + "input_tokens": 1145388, + "output_tokens": 3448, + "total_tokens": 1148836, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 360.072603225708, + "token_usage": { + "input_tokens": 474957, + "output_tokens": 4516, + "total_tokens": 479473, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 257.3946008682251, + "token_usage": { + "input_tokens": 1001536, + "output_tokens": 4424, + "total_tokens": 1005960, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/threestudio__code_locating.json b/model_results/glm-4-6/threestudio__code_locating.json new file mode 100644 index 00000000..2c7d3644 --- /dev/null +++ b/model_results/glm-4-6/threestudio__code_locating.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "threestudio__code_locating", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 26.136942386627197, + "token_usage": { + "input_tokens": 51573, + "output_tokens": 1646, + "total_tokens": 53219, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 17.75987672805786, + "token_usage": { + "input_tokens": 32672, + "output_tokens": 1174, + "total_tokens": 33846, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 26.136942386627197, + "token_usage": { + "input_tokens": 51573, + "output_tokens": 1646, + "total_tokens": 53219, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 17.75987672805786, + "token_usage": { + "input_tokens": 32672, + "output_tokens": 1174, + "total_tokens": 33846, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/threestudio__output_analysis.json b/model_results/glm-4-6/threestudio__output_analysis.json new file mode 100644 index 00000000..9f2b12f0 --- /dev/null +++ b/model_results/glm-4-6/threestudio__output_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "threestudio__output_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 22.722601413726807, + "token_usage": { + "input_tokens": 77786, + "output_tokens": 1572, + "total_tokens": 79358, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 58.15586066246033, + "token_usage": { + "input_tokens": 56720, + "output_tokens": 1464, + "total_tokens": 58184, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 22.722601413726807, + "token_usage": { + "input_tokens": 77786, + "output_tokens": 1572, + "total_tokens": 79358, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 58.15586066246033, + "token_usage": { + "input_tokens": 56720, + "output_tokens": 1464, + "total_tokens": 58184, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/threestudio__requirements_completion.json b/model_results/glm-4-6/threestudio__requirements_completion.json new file mode 100644 index 00000000..1577f06c --- /dev/null +++ b/model_results/glm-4-6/threestudio__requirements_completion.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "threestudio__requirements_completion", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 38.238969564437866, + "token_usage": { + "input_tokens": 246150, + "output_tokens": 1918, + "total_tokens": 248068, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 32.380354166030884, + "token_usage": { + "input_tokens": 263092, + "output_tokens": 2145, + "total_tokens": 265237, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 38.238969564437866, + "token_usage": { + "input_tokens": 246150, + "output_tokens": 1918, + "total_tokens": 248068, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 32.380354166030884, + "token_usage": { + "input_tokens": 263092, + "output_tokens": 2145, + "total_tokens": 265237, + "reasoning_tokens": 0 + }, + "turn_count": 16 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/toronto_guide__change_color.json b/model_results/glm-4-6/toronto_guide__change_color.json new file mode 100644 index 00000000..1c2e11cf --- /dev/null +++ b/model_results/glm-4-6/toronto_guide__change_color.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "toronto_guide__change_color", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 316.3442397117615, + "token_usage": { + "input_tokens": 1248337, + "output_tokens": 7434, + "total_tokens": 1255771, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + "run-2": { + "success": false, + "error_message": "Expecting value: line 1 column 104 (char 103)", + "execution_time": 40.342220306396484, + "token_usage": { + "input_tokens": 151525, + "output_tokens": 786, + "total_tokens": 152311, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": "Expecting value: line 1 column 59 (char 58)", + "execution_time": 510.50904059410095, + "token_usage": { + "input_tokens": 1357240, + "output_tokens": 7605, + "total_tokens": 1364845, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 521.605447769165, + "token_usage": { + "input_tokens": 1591205, + "output_tokens": 10707, + "total_tokens": 1601912, + "reasoning_tokens": 0 + }, + "turn_count": 31 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/toronto_guide__weekend_adventure_planner.json b/model_results/glm-4-6/toronto_guide__weekend_adventure_planner.json new file mode 100644 index 00000000..f056ba04 --- /dev/null +++ b/model_results/glm-4-6/toronto_guide__weekend_adventure_planner.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "notion", + "task": "toronto_guide__weekend_adventure_planner", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 89.0496027469635, + "token_usage": { + "input_tokens": 179357, + "output_tokens": 2434, + "total_tokens": 181791, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": "Max turns (100) exceeded", + "execution_time": 807.9166803359985, + "token_usage": { + "input_tokens": 5091783, + "output_tokens": 11537, + "total_tokens": 5103320, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 139.65669798851013, + "token_usage": { + "input_tokens": 114812, + "output_tokens": 2683, + "total_tokens": 117495, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 206.51325869560242, + "token_usage": { + "input_tokens": 679223, + "output_tokens": 3495, + "total_tokens": 682718, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/vectors__dba_vector_analysis.json b/model_results/glm-4-6/vectors__dba_vector_analysis.json new file mode 100644 index 00000000..ae94b2a3 --- /dev/null +++ b/model_results/glm-4-6/vectors__dba_vector_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "postgres", + "task": "vectors__dba_vector_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 367.1814503669739, + "token_usage": { + "input_tokens": 378662, + "output_tokens": 6271, + "total_tokens": 384933, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 398.6988573074341, + "token_usage": { + "input_tokens": 370868, + "output_tokens": 5668, + "total_tokens": 376536, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 258.43459033966064, + "token_usage": { + "input_tokens": 445683, + "output_tokens": 7381, + "total_tokens": 453064, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 439.8751804828644, + "token_usage": { + "input_tokens": 519258, + "output_tokens": 8700, + "total_tokens": 527958, + "reasoning_tokens": 0 + }, + "turn_count": 36 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/votenet__dataset_comparison.json b/model_results/glm-4-6/votenet__dataset_comparison.json new file mode 100644 index 00000000..b099dd57 --- /dev/null +++ b/model_results/glm-4-6/votenet__dataset_comparison.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "votenet__dataset_comparison", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 34.750967025756836, + "token_usage": { + "input_tokens": 143277, + "output_tokens": 1019, + "total_tokens": 144296, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 32.7380051612854, + "token_usage": { + "input_tokens": 181935, + "output_tokens": 1234, + "total_tokens": 183169, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 34.750967025756836, + "token_usage": { + "input_tokens": 143277, + "output_tokens": 1019, + "total_tokens": 144296, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 32.7380051612854, + "token_usage": { + "input_tokens": 181935, + "output_tokens": 1234, + "total_tokens": 183169, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/votenet__debugging.json b/model_results/glm-4-6/votenet__debugging.json new file mode 100644 index 00000000..468c4d22 --- /dev/null +++ b/model_results/glm-4-6/votenet__debugging.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "votenet__debugging", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 41.36724042892456, + "token_usage": { + "input_tokens": 47588, + "output_tokens": 1760, + "total_tokens": 49348, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 39.127994775772095, + "token_usage": { + "input_tokens": 217571, + "output_tokens": 2389, + "total_tokens": 219960, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 41.36724042892456, + "token_usage": { + "input_tokens": 47588, + "output_tokens": 1760, + "total_tokens": 49348, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 39.127994775772095, + "token_usage": { + "input_tokens": 217571, + "output_tokens": 2389, + "total_tokens": 219960, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/votenet__requirements_writing.json b/model_results/glm-4-6/votenet__requirements_writing.json new file mode 100644 index 00000000..c26f77b9 --- /dev/null +++ b/model_results/glm-4-6/votenet__requirements_writing.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "filesystem", + "task": "votenet__requirements_writing", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 51.30520844459534, + "token_usage": { + "input_tokens": 247922, + "output_tokens": 2129, + "total_tokens": 250051, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 32.39541029930115, + "token_usage": { + "input_tokens": 244035, + "output_tokens": 2274, + "total_tokens": 246309, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 51.30520844459534, + "token_usage": { + "input_tokens": 247922, + "output_tokens": 2129, + "total_tokens": 250051, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 32.39541029930115, + "token_usage": { + "input_tokens": 244035, + "output_tokens": 2274, + "total_tokens": 246309, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/web_search__birth_of_arvinxu.json b/model_results/glm-4-6/web_search__birth_of_arvinxu.json new file mode 100644 index 00000000..87c8316a --- /dev/null +++ b/model_results/glm-4-6/web_search__birth_of_arvinxu.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "web_search__birth_of_arvinxu", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 146.43275094032288, + "token_usage": { + "input_tokens": 664824, + "output_tokens": 1972, + "total_tokens": 666796, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 116.21653485298157, + "token_usage": { + "input_tokens": 670862, + "output_tokens": 2334, + "total_tokens": 673196, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 269546 tokens (266928 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 52.30887579917908, + "token_usage": { + "input_tokens": 60062, + "output_tokens": 984, + "total_tokens": 61046, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 214.3152141571045, + "token_usage": { + "input_tokens": 1061338, + "output_tokens": 2960, + "total_tokens": 1064298, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/glm-4-6/web_search__r1_arxiv.json b/model_results/glm-4-6/web_search__r1_arxiv.json new file mode 100644 index 00000000..2882a019 --- /dev/null +++ b/model_results/glm-4-6/web_search__r1_arxiv.json @@ -0,0 +1,55 @@ +{ + "model": "glm-4-6", + "service": "playwright", + "task": "web_search__r1_arxiv", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 139.40616869926453, + "token_usage": { + "input_tokens": 707367, + "output_tokens": 1108, + "total_tokens": 708475, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 220.23436331748962, + "token_usage": { + "input_tokens": 731476, + "output_tokens": 4505, + "total_tokens": 735981, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 205087 tokens (202469 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 204.28891038894653, + "token_usage": { + "input_tokens": 904752, + "output_tokens": 1548, + "total_tokens": 906300, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": "litellm.BadRequestError: OpenrouterException - {\"error\":{\"message\":\"This endpoint's maximum context length is 204800 tokens. However, you requested about 220819 tokens (218201 of text input, 2618 of tool input). Please reduce the length of either one, or use the \\\"middle-out\\\" transform to compress your prompt automatically.\",\"code\":400,\"metadata\":{\"provider_name\":null}}}", + "execution_time": 139.04804301261902, + "token_usage": { + "input_tokens": 550098, + "output_tokens": 962, + "total_tokens": 551060, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/build_your_own_x__find_commit_date.json b/model_results/grok-4-fast/build_your_own_x__find_commit_date.json new file mode 100644 index 00000000..e1a94e0e --- /dev/null +++ b/model_results/grok-4-fast/build_your_own_x__find_commit_date.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "build_your_own_x__find_commit_date", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 2173.348979949951, + "token_usage": { + "input_tokens": 10285265, + "output_tokens": 128617, + "total_tokens": 10413882, + "reasoning_tokens": 7454 + }, + "turn_count": 40 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 82.50000667572021, + "token_usage": { + "input_tokens": 145342, + "output_tokens": 7883, + "total_tokens": 153225, + "reasoning_tokens": 2148 + }, + "turn_count": 8 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 130.41381669044495, + "token_usage": { + "input_tokens": 1375606, + "output_tokens": 6358, + "total_tokens": 1381964, + "reasoning_tokens": 5206 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 1340.6879160404205, + "token_usage": { + "input_tokens": 1283786, + "output_tokens": 120609, + "total_tokens": 1404395, + "reasoning_tokens": 2479 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/build_your_own_x__find_rag_commit.json b/model_results/grok-4-fast/build_your_own_x__find_rag_commit.json new file mode 100644 index 00000000..24d80a75 --- /dev/null +++ b/model_results/grok-4-fast/build_your_own_x__find_rag_commit.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "build_your_own_x__find_rag_commit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 77.67906475067139, + "token_usage": { + "input_tokens": 641675, + "output_tokens": 4859, + "total_tokens": 646534, + "reasoning_tokens": 1672 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 104.13842034339905, + "token_usage": { + "input_tokens": 459613, + "output_tokens": 7085, + "total_tokens": 466698, + "reasoning_tokens": 2563 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1076.2166833877563, + "token_usage": { + "input_tokens": 785061, + "output_tokens": 101054, + "total_tokens": 886115, + "reasoning_tokens": 2377 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 121.34236764907837, + "token_usage": { + "input_tokens": 818050, + "output_tokens": 7075, + "total_tokens": 825125, + "reasoning_tokens": 5364 + }, + "turn_count": 33 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/chinook__customer_data_migration.json b/model_results/grok-4-fast/chinook__customer_data_migration.json new file mode 100644 index 00000000..3ed8c2d7 --- /dev/null +++ b/model_results/grok-4-fast/chinook__customer_data_migration.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "chinook__customer_data_migration", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 107.74598097801208, + "token_usage": { + "input_tokens": 207094, + "output_tokens": 13978, + "total_tokens": 221072, + "reasoning_tokens": 1412 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 100.56838035583496, + "token_usage": { + "input_tokens": 267760, + "output_tokens": 15491, + "total_tokens": 283251, + "reasoning_tokens": 3363 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 107.42437553405762, + "token_usage": { + "input_tokens": 101514, + "output_tokens": 14321, + "total_tokens": 115835, + "reasoning_tokens": 1880 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 44.38723421096802, + "token_usage": { + "input_tokens": 61616, + "output_tokens": 4486, + "total_tokens": 66102, + "reasoning_tokens": 4315 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/chinook__employee_hierarchy_management.json b/model_results/grok-4-fast/chinook__employee_hierarchy_management.json new file mode 100644 index 00000000..f890f24c --- /dev/null +++ b/model_results/grok-4-fast/chinook__employee_hierarchy_management.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "chinook__employee_hierarchy_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 47.389010190963745, + "token_usage": { + "input_tokens": 122294, + "output_tokens": 3222, + "total_tokens": 125516, + "reasoning_tokens": 1748 + }, + "turn_count": 23 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 46.88154339790344, + "token_usage": { + "input_tokens": 118297, + "output_tokens": 3777, + "total_tokens": 122074, + "reasoning_tokens": 2361 + }, + "turn_count": 20 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 71.06090593338013, + "token_usage": { + "input_tokens": 240727, + "output_tokens": 4382, + "total_tokens": 245109, + "reasoning_tokens": 2761 + }, + "turn_count": 28 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 41.21628499031067, + "token_usage": { + "input_tokens": 101924, + "output_tokens": 4250, + "total_tokens": 106174, + "reasoning_tokens": 3127 + }, + "turn_count": 16 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/chinook__sales_and_music_charts.json b/model_results/grok-4-fast/chinook__sales_and_music_charts.json new file mode 100644 index 00000000..c6d5c9d0 --- /dev/null +++ b/model_results/grok-4-fast/chinook__sales_and_music_charts.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "chinook__sales_and_music_charts", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 52.323203325271606, + "token_usage": { + "input_tokens": 175397, + "output_tokens": 4181, + "total_tokens": 179578, + "reasoning_tokens": 2682 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 41.52652311325073, + "token_usage": { + "input_tokens": 112951, + "output_tokens": 4029, + "total_tokens": 116980, + "reasoning_tokens": 2467 + }, + "turn_count": 15 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 32.15245079994202, + "token_usage": { + "input_tokens": 112270, + "output_tokens": 2676, + "total_tokens": 114946, + "reasoning_tokens": 1130 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 35.22850847244263, + "token_usage": { + "input_tokens": 66931, + "output_tokens": 5081, + "total_tokens": 72012, + "reasoning_tokens": 3887 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/claude-code__automated_changelog_generation.json b/model_results/grok-4-fast/claude-code__automated_changelog_generation.json new file mode 100644 index 00000000..76fe1399 --- /dev/null +++ b/model_results/grok-4-fast/claude-code__automated_changelog_generation.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "claude-code__automated_changelog_generation", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 56.097662925720215, + "token_usage": { + "input_tokens": 140723, + "output_tokens": 4328, + "total_tokens": 145051, + "reasoning_tokens": 2169 + }, + "turn_count": 4 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 49.320733070373535, + "token_usage": { + "input_tokens": 180963, + "output_tokens": 4340, + "total_tokens": 185303, + "reasoning_tokens": 2196 + }, + "turn_count": 5 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 46.808067083358765, + "token_usage": { + "input_tokens": 144449, + "output_tokens": 5246, + "total_tokens": 149695, + "reasoning_tokens": 3258 + }, + "turn_count": 4 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 66.99075412750244, + "token_usage": { + "input_tokens": 185793, + "output_tokens": 6143, + "total_tokens": 191936, + "reasoning_tokens": 3966 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/claude-code__claude_collaboration_analysis.json b/model_results/grok-4-fast/claude-code__claude_collaboration_analysis.json new file mode 100644 index 00000000..245bc565 --- /dev/null +++ b/model_results/grok-4-fast/claude-code__claude_collaboration_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "claude-code__claude_collaboration_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 28.926344871520996, + "token_usage": { + "input_tokens": 99563, + "output_tokens": 2638, + "total_tokens": 102201, + "reasoning_tokens": 2306 + }, + "turn_count": 4 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 32.524813413619995, + "token_usage": { + "input_tokens": 163099, + "output_tokens": 2263, + "total_tokens": 165362, + "reasoning_tokens": 1820 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 31.306444883346558, + "token_usage": { + "input_tokens": 113851, + "output_tokens": 2170, + "total_tokens": 116021, + "reasoning_tokens": 1810 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 43.91437554359436, + "token_usage": { + "input_tokens": 149866, + "output_tokens": 2950, + "total_tokens": 152816, + "reasoning_tokens": 2571 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/claude-code__critical_issue_hotfix_workflow.json b/model_results/grok-4-fast/claude-code__critical_issue_hotfix_workflow.json new file mode 100644 index 00000000..f0378938 --- /dev/null +++ b/model_results/grok-4-fast/claude-code__critical_issue_hotfix_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "claude-code__critical_issue_hotfix_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 53.511555194854736, + "token_usage": { + "input_tokens": 73729, + "output_tokens": 4000, + "total_tokens": 77729, + "reasoning_tokens": 1834 + }, + "turn_count": 4 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 47.90933275222778, + "token_usage": { + "input_tokens": 218094, + "output_tokens": 3410, + "total_tokens": 221504, + "reasoning_tokens": 1883 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 72.52073359489441, + "token_usage": { + "input_tokens": 292221, + "output_tokens": 4537, + "total_tokens": 296758, + "reasoning_tokens": 2442 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 50.27313423156738, + "token_usage": { + "input_tokens": 216188, + "output_tokens": 3135, + "total_tokens": 219323, + "reasoning_tokens": 1475 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/claude-code__feature_commit_tracking.json b/model_results/grok-4-fast/claude-code__feature_commit_tracking.json new file mode 100644 index 00000000..df79a543 --- /dev/null +++ b/model_results/grok-4-fast/claude-code__feature_commit_tracking.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "claude-code__feature_commit_tracking", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 76.72193765640259, + "token_usage": { + "input_tokens": 512304, + "output_tokens": 6269, + "total_tokens": 518573, + "reasoning_tokens": 3420 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 117.03881025314331, + "token_usage": { + "input_tokens": 1127110, + "output_tokens": 6935, + "total_tokens": 1134045, + "reasoning_tokens": 3869 + }, + "turn_count": 23 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 88.72263360023499, + "token_usage": { + "input_tokens": 548929, + "output_tokens": 5501, + "total_tokens": 554430, + "reasoning_tokens": 2861 + }, + "turn_count": 19 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 64.88823890686035, + "token_usage": { + "input_tokens": 444895, + "output_tokens": 4457, + "total_tokens": 449352, + "reasoning_tokens": 2426 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/claude-code__label_color_standardization.json b/model_results/grok-4-fast/claude-code__label_color_standardization.json new file mode 100644 index 00000000..cb18552d --- /dev/null +++ b/model_results/grok-4-fast/claude-code__label_color_standardization.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "claude-code__label_color_standardization", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 101.20335292816162, + "token_usage": { + "input_tokens": 335088, + "output_tokens": 7304, + "total_tokens": 342392, + "reasoning_tokens": 4015 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 373.88942766189575, + "token_usage": { + "input_tokens": 442298, + "output_tokens": 28200, + "total_tokens": 470498, + "reasoning_tokens": 6453 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 84.03827977180481, + "token_usage": { + "input_tokens": 831597, + "output_tokens": 6154, + "total_tokens": 837751, + "reasoning_tokens": 4240 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 113.16027569770813, + "token_usage": { + "input_tokens": 747581, + "output_tokens": 6563, + "total_tokens": 754144, + "reasoning_tokens": 4657 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/company_in_a_box__employee_onboarding.json b/model_results/grok-4-fast/company_in_a_box__employee_onboarding.json new file mode 100644 index 00000000..ce12a5ab --- /dev/null +++ b/model_results/grok-4-fast/company_in_a_box__employee_onboarding.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "company_in_a_box__employee_onboarding", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 152.7811119556427, + "token_usage": { + "input_tokens": 370290, + "output_tokens": 10484, + "total_tokens": 380774, + "reasoning_tokens": 7985 + }, + "turn_count": 30 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 82.09955382347107, + "token_usage": { + "input_tokens": 232701, + "output_tokens": 7011, + "total_tokens": 239712, + "reasoning_tokens": 5431 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 100.55681276321411, + "token_usage": { + "input_tokens": 251647, + "output_tokens": 9730, + "total_tokens": 261377, + "reasoning_tokens": 3990 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 111.66854643821716, + "token_usage": { + "input_tokens": 213459, + "output_tokens": 9190, + "total_tokens": 222649, + "reasoning_tokens": 7244 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/company_in_a_box__goals_restructure.json b/model_results/grok-4-fast/company_in_a_box__goals_restructure.json new file mode 100644 index 00000000..39852610 --- /dev/null +++ b/model_results/grok-4-fast/company_in_a_box__goals_restructure.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "company_in_a_box__goals_restructure", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 89.88013482093811, + "token_usage": { + "input_tokens": 265275, + "output_tokens": 7482, + "total_tokens": 272757, + "reasoning_tokens": 6539 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 73.29062843322754, + "token_usage": { + "input_tokens": 184815, + "output_tokens": 7497, + "total_tokens": 192312, + "reasoning_tokens": 6548 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 81.7825391292572, + "token_usage": { + "input_tokens": 233964, + "output_tokens": 6802, + "total_tokens": 240766, + "reasoning_tokens": 5624 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 71.3629059791565, + "token_usage": { + "input_tokens": 142508, + "output_tokens": 8480, + "total_tokens": 150988, + "reasoning_tokens": 7512 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/company_in_a_box__quarterly_review_dashboard.json b/model_results/grok-4-fast/company_in_a_box__quarterly_review_dashboard.json new file mode 100644 index 00000000..38cbe9c8 --- /dev/null +++ b/model_results/grok-4-fast/company_in_a_box__quarterly_review_dashboard.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "company_in_a_box__quarterly_review_dashboard", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 104.81969857215881, + "token_usage": { + "input_tokens": 312438, + "output_tokens": 6789, + "total_tokens": 319227, + "reasoning_tokens": 5274 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 592.4784758090973, + "token_usage": { + "input_tokens": 64955, + "output_tokens": 117513, + "total_tokens": 182468, + "reasoning_tokens": 2823 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 86.93055653572083, + "token_usage": { + "input_tokens": 263949, + "output_tokens": 6796, + "total_tokens": 270745, + "reasoning_tokens": 4090 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 173.68438863754272, + "token_usage": { + "input_tokens": 960578, + "output_tokens": 15556, + "total_tokens": 976134, + "reasoning_tokens": 8531 + }, + "turn_count": 41 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/computer_science_student_dashboard__code_snippets_go.json b/model_results/grok-4-fast/computer_science_student_dashboard__code_snippets_go.json new file mode 100644 index 00000000..e2b268f3 --- /dev/null +++ b/model_results/grok-4-fast/computer_science_student_dashboard__code_snippets_go.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "computer_science_student_dashboard__code_snippets_go", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 152.2839071750641, + "token_usage": { + "input_tokens": 431250, + "output_tokens": 6865, + "total_tokens": 438115, + "reasoning_tokens": 5144 + }, + "turn_count": 22 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 90.05319833755493, + "token_usage": { + "input_tokens": 338050, + "output_tokens": 5862, + "total_tokens": 343912, + "reasoning_tokens": 4509 + }, + "turn_count": 20 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 72.62897086143494, + "token_usage": { + "input_tokens": 442974, + "output_tokens": 5445, + "total_tokens": 448419, + "reasoning_tokens": 3707 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 117.43509840965271, + "token_usage": { + "input_tokens": 506019, + "output_tokens": 9186, + "total_tokens": 515205, + "reasoning_tokens": 7181 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/computer_science_student_dashboard__courses_internships_relation.json b/model_results/grok-4-fast/computer_science_student_dashboard__courses_internships_relation.json new file mode 100644 index 00000000..00d9eb80 --- /dev/null +++ b/model_results/grok-4-fast/computer_science_student_dashboard__courses_internships_relation.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "computer_science_student_dashboard__courses_internships_relation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 137.83059763908386, + "token_usage": { + "input_tokens": 746141, + "output_tokens": 10248, + "total_tokens": 756389, + "reasoning_tokens": 7360 + }, + "turn_count": 32 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 108.66051697731018, + "token_usage": { + "input_tokens": 496192, + "output_tokens": 8675, + "total_tokens": 504867, + "reasoning_tokens": 6730 + }, + "turn_count": 28 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 118.29310750961304, + "token_usage": { + "input_tokens": 338355, + "output_tokens": 8281, + "total_tokens": 346636, + "reasoning_tokens": 6435 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 147.53497695922852, + "token_usage": { + "input_tokens": 639194, + "output_tokens": 11619, + "total_tokens": 650813, + "reasoning_tokens": 8596 + }, + "turn_count": 32 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/computer_science_student_dashboard__study_session_tracker.json b/model_results/grok-4-fast/computer_science_student_dashboard__study_session_tracker.json new file mode 100644 index 00000000..605b87d5 --- /dev/null +++ b/model_results/grok-4-fast/computer_science_student_dashboard__study_session_tracker.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "computer_science_student_dashboard__study_session_tracker", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 734.4401268959045, + "token_usage": { + "input_tokens": 559753, + "output_tokens": 8346, + "total_tokens": 568099, + "reasoning_tokens": 5860 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 596.5692698955536, + "token_usage": { + "input_tokens": 104787, + "output_tokens": 114695, + "total_tokens": 219482, + "reasoning_tokens": 4280 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 106.04337930679321, + "token_usage": { + "input_tokens": 148630, + "output_tokens": 15641, + "total_tokens": 164271, + "reasoning_tokens": 6369 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 70.91306114196777, + "token_usage": { + "input_tokens": 258821, + "output_tokens": 6318, + "total_tokens": 265139, + "reasoning_tokens": 4860 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/desktop__music_report.json b/model_results/grok-4-fast/desktop__music_report.json new file mode 100644 index 00000000..eeff6272 --- /dev/null +++ b/model_results/grok-4-fast/desktop__music_report.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "desktop__music_report", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 42.31699299812317, + "token_usage": { + "input_tokens": 38096, + "output_tokens": 5484, + "total_tokens": 43580, + "reasoning_tokens": 4929 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 47.07411599159241, + "token_usage": { + "input_tokens": 37741, + "output_tokens": 5306, + "total_tokens": 43047, + "reasoning_tokens": 4697 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 38.21270203590393, + "token_usage": { + "input_tokens": 36783, + "output_tokens": 4806, + "total_tokens": 41589, + "reasoning_tokens": 4198 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 41.401585817337036, + "token_usage": { + "input_tokens": 37032, + "output_tokens": 4835, + "total_tokens": 41867, + "reasoning_tokens": 4226 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/desktop__project_management.json b/model_results/grok-4-fast/desktop__project_management.json new file mode 100644 index 00000000..29279655 --- /dev/null +++ b/model_results/grok-4-fast/desktop__project_management.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "desktop__project_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 106.77428698539734, + "token_usage": { + "input_tokens": 171223, + "output_tokens": 10862, + "total_tokens": 182085, + "reasoning_tokens": 5294 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 65.06278777122498, + "token_usage": { + "input_tokens": 188413, + "output_tokens": 9143, + "total_tokens": 197556, + "reasoning_tokens": 4336 + }, + "turn_count": 20 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 85.98844456672668, + "token_usage": { + "input_tokens": 198448, + "output_tokens": 10371, + "total_tokens": 208819, + "reasoning_tokens": 5337 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 75.20499229431152, + "token_usage": { + "input_tokens": 198162, + "output_tokens": 7347, + "total_tokens": 205509, + "reasoning_tokens": 3147 + }, + "turn_count": 28 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/desktop__timeline_extraction.json b/model_results/grok-4-fast/desktop__timeline_extraction.json new file mode 100644 index 00000000..1810518b --- /dev/null +++ b/model_results/grok-4-fast/desktop__timeline_extraction.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "desktop__timeline_extraction", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 49.41848683357239, + "token_usage": { + "input_tokens": 46409, + "output_tokens": 6445, + "total_tokens": 52854, + "reasoning_tokens": 4977 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 40.88468360900879, + "token_usage": { + "input_tokens": 64128, + "output_tokens": 7290, + "total_tokens": 71418, + "reasoning_tokens": 5558 + }, + "turn_count": 7 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 50.150161266326904, + "token_usage": { + "input_tokens": 63016, + "output_tokens": 8631, + "total_tokens": 71647, + "reasoning_tokens": 6849 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 69.8250961303711, + "token_usage": { + "input_tokens": 355044, + "output_tokens": 6844, + "total_tokens": 361888, + "reasoning_tokens": 4666 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/desktop_template__budget_computation.json b/model_results/grok-4-fast/desktop_template__budget_computation.json new file mode 100644 index 00000000..918438a1 --- /dev/null +++ b/model_results/grok-4-fast/desktop_template__budget_computation.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "desktop_template__budget_computation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 33.91645622253418, + "token_usage": { + "input_tokens": 94530, + "output_tokens": 2658, + "total_tokens": 97188, + "reasoning_tokens": 1589 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 31.894129037857056, + "token_usage": { + "input_tokens": 55870, + "output_tokens": 4269, + "total_tokens": 60139, + "reasoning_tokens": 3147 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 43.58357357978821, + "token_usage": { + "input_tokens": 50029, + "output_tokens": 4146, + "total_tokens": 54175, + "reasoning_tokens": 2983 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 45.23974370956421, + "token_usage": { + "input_tokens": 95833, + "output_tokens": 4472, + "total_tokens": 100305, + "reasoning_tokens": 3416 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/desktop_template__contact_information.json b/model_results/grok-4-fast/desktop_template__contact_information.json new file mode 100644 index 00000000..7f9b894e --- /dev/null +++ b/model_results/grok-4-fast/desktop_template__contact_information.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "desktop_template__contact_information", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 92.011953830719, + "token_usage": { + "input_tokens": 213694, + "output_tokens": 7413, + "total_tokens": 221107, + "reasoning_tokens": 5623 + }, + "turn_count": 31 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 54.116535902023315, + "token_usage": { + "input_tokens": 137809, + "output_tokens": 6489, + "total_tokens": 144298, + "reasoning_tokens": 4809 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 78.22228336334229, + "token_usage": { + "input_tokens": 229375, + "output_tokens": 5489, + "total_tokens": 234864, + "reasoning_tokens": 3272 + }, + "turn_count": 38 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 59.964751958847046, + "token_usage": { + "input_tokens": 118938, + "output_tokens": 5836, + "total_tokens": 124774, + "reasoning_tokens": 4514 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/desktop_template__file_arrangement.json b/model_results/grok-4-fast/desktop_template__file_arrangement.json new file mode 100644 index 00000000..9edff0d8 --- /dev/null +++ b/model_results/grok-4-fast/desktop_template__file_arrangement.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "desktop_template__file_arrangement", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 52.440882205963135, + "token_usage": { + "input_tokens": 80407, + "output_tokens": 5584, + "total_tokens": 85991, + "reasoning_tokens": 2423 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 41.556867361068726, + "token_usage": { + "input_tokens": 29973, + "output_tokens": 4927, + "total_tokens": 34900, + "reasoning_tokens": 2251 + }, + "turn_count": 6 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 59.65416622161865, + "token_usage": { + "input_tokens": 91228, + "output_tokens": 8705, + "total_tokens": 99933, + "reasoning_tokens": 4938 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 51.68978476524353, + "token_usage": { + "input_tokens": 130077, + "output_tokens": 6596, + "total_tokens": 136673, + "reasoning_tokens": 3662 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/dvdrental__customer_analysis_fix.json b/model_results/grok-4-fast/dvdrental__customer_analysis_fix.json new file mode 100644 index 00000000..c47f8827 --- /dev/null +++ b/model_results/grok-4-fast/dvdrental__customer_analysis_fix.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "dvdrental__customer_analysis_fix", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 114.13974523544312, + "token_usage": { + "input_tokens": 259197, + "output_tokens": 12745, + "total_tokens": 271942, + "reasoning_tokens": 10039 + }, + "turn_count": 26 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 104.39919996261597, + "token_usage": { + "input_tokens": 571531, + "output_tokens": 9943, + "total_tokens": 581474, + "reasoning_tokens": 6295 + }, + "turn_count": 15 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 111.87351703643799, + "token_usage": { + "input_tokens": 258907, + "output_tokens": 12764, + "total_tokens": 271671, + "reasoning_tokens": 6973 + }, + "turn_count": 23 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 114.15665888786316, + "token_usage": { + "input_tokens": 234116, + "output_tokens": 10917, + "total_tokens": 245033, + "reasoning_tokens": 8227 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/dvdrental__customer_analytics_optimization.json b/model_results/grok-4-fast/dvdrental__customer_analytics_optimization.json new file mode 100644 index 00000000..425bf5a6 --- /dev/null +++ b/model_results/grok-4-fast/dvdrental__customer_analytics_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "dvdrental__customer_analytics_optimization", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 60.74306917190552, + "token_usage": { + "input_tokens": 118735, + "output_tokens": 6359, + "total_tokens": 125094, + "reasoning_tokens": 2249 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 58.639387369155884, + "token_usage": { + "input_tokens": 45077, + "output_tokens": 5332, + "total_tokens": 50409, + "reasoning_tokens": 2368 + }, + "turn_count": 7 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 51.7120943069458, + "token_usage": { + "input_tokens": 257960, + "output_tokens": 5017, + "total_tokens": 262977, + "reasoning_tokens": 2595 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 39.97567844390869, + "token_usage": { + "input_tokens": 10960, + "output_tokens": 3703, + "total_tokens": 14663, + "reasoning_tokens": 1325 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/dvdrental__film_inventory_management.json b/model_results/grok-4-fast/dvdrental__film_inventory_management.json new file mode 100644 index 00000000..56536657 --- /dev/null +++ b/model_results/grok-4-fast/dvdrental__film_inventory_management.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "dvdrental__film_inventory_management", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 26.48506808280945, + "token_usage": { + "input_tokens": 49443, + "output_tokens": 3283, + "total_tokens": 52726, + "reasoning_tokens": 2477 + }, + "turn_count": 10 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 130.48151683807373, + "token_usage": { + "input_tokens": 18333, + "output_tokens": 5529, + "total_tokens": 23862, + "reasoning_tokens": 4595 + }, + "turn_count": 3 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 37.734288930892944, + "token_usage": { + "input_tokens": 44486, + "output_tokens": 2901, + "total_tokens": 47387, + "reasoning_tokens": 2003 + }, + "turn_count": 11 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 25.89009380340576, + "token_usage": { + "input_tokens": 33609, + "output_tokens": 2442, + "total_tokens": 36051, + "reasoning_tokens": 1533 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/easyr1__advanced_branch_strategy.json b/model_results/grok-4-fast/easyr1__advanced_branch_strategy.json new file mode 100644 index 00000000..129de589 --- /dev/null +++ b/model_results/grok-4-fast/easyr1__advanced_branch_strategy.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "easyr1__advanced_branch_strategy", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 57.045555114746094, + "token_usage": { + "input_tokens": 82074, + "output_tokens": 2965, + "total_tokens": 85039, + "reasoning_tokens": 2011 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 67.60392880439758, + "token_usage": { + "input_tokens": 95293, + "output_tokens": 6425, + "total_tokens": 101718, + "reasoning_tokens": 5102 + }, + "turn_count": 5 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 58.84398698806763, + "token_usage": { + "input_tokens": 181063, + "output_tokens": 4678, + "total_tokens": 185741, + "reasoning_tokens": 3850 + }, + "turn_count": 10 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 57.27000713348389, + "token_usage": { + "input_tokens": 163654, + "output_tokens": 2875, + "total_tokens": 166529, + "reasoning_tokens": 2034 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/easyr1__config_parameter_audit.json b/model_results/grok-4-fast/easyr1__config_parameter_audit.json new file mode 100644 index 00000000..56df7f0f --- /dev/null +++ b/model_results/grok-4-fast/easyr1__config_parameter_audit.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "easyr1__config_parameter_audit", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 93.72474551200867, + "token_usage": { + "input_tokens": 527993, + "output_tokens": 6246, + "total_tokens": 534239, + "reasoning_tokens": 4365 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 1240.415374994278, + "token_usage": { + "input_tokens": 3755379, + "output_tokens": 111412, + "total_tokens": 3866791, + "reasoning_tokens": 3063 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 101.83688473701477, + "token_usage": { + "input_tokens": 239032, + "output_tokens": 7199, + "total_tokens": 246231, + "reasoning_tokens": 2046 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 66.84354496002197, + "token_usage": { + "input_tokens": 373114, + "output_tokens": 3853, + "total_tokens": 376967, + "reasoning_tokens": 2718 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/easyr1__performance_regression_investigation.json b/model_results/grok-4-fast/easyr1__performance_regression_investigation.json new file mode 100644 index 00000000..8a8288b3 --- /dev/null +++ b/model_results/grok-4-fast/easyr1__performance_regression_investigation.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "easyr1__performance_regression_investigation", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 44.879680156707764, + "token_usage": { + "input_tokens": 56262, + "output_tokens": 3113, + "total_tokens": 59375, + "reasoning_tokens": 1438 + }, + "turn_count": 3 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 75.66387295722961, + "token_usage": { + "input_tokens": 79241, + "output_tokens": 5587, + "total_tokens": 84828, + "reasoning_tokens": 4018 + }, + "turn_count": 4 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 332.8027970790863, + "token_usage": { + "input_tokens": 290569, + "output_tokens": 13855, + "total_tokens": 304424, + "reasoning_tokens": 2432 + }, + "turn_count": 9 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 39.77624320983887, + "token_usage": { + "input_tokens": 99935, + "output_tokens": 3485, + "total_tokens": 103420, + "reasoning_tokens": 1919 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/easyr1__qwen3_issue_management.json b/model_results/grok-4-fast/easyr1__qwen3_issue_management.json new file mode 100644 index 00000000..0be6fc8d --- /dev/null +++ b/model_results/grok-4-fast/easyr1__qwen3_issue_management.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "easyr1__qwen3_issue_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 65.41552782058716, + "token_usage": { + "input_tokens": 1028466, + "output_tokens": 2069, + "total_tokens": 1030535, + "reasoning_tokens": 1127 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 80.247314453125, + "token_usage": { + "input_tokens": 1530229, + "output_tokens": 1578, + "total_tokens": 1531807, + "reasoning_tokens": 1290 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 96.00932621955872, + "token_usage": { + "input_tokens": 1751625, + "output_tokens": 3825, + "total_tokens": 1755450, + "reasoning_tokens": 2569 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 63.5764684677124, + "token_usage": { + "input_tokens": 237245, + "output_tokens": 3853, + "total_tokens": 241098, + "reasoning_tokens": 2441 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/employees__employee_demographics_report.json b/model_results/grok-4-fast/employees__employee_demographics_report.json new file mode 100644 index 00000000..19bbc237 --- /dev/null +++ b/model_results/grok-4-fast/employees__employee_demographics_report.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "employees__employee_demographics_report", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 37.23557090759277, + "token_usage": { + "input_tokens": 65053, + "output_tokens": 3774, + "total_tokens": 68827, + "reasoning_tokens": 2399 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 52.087852478027344, + "token_usage": { + "input_tokens": 107073, + "output_tokens": 6365, + "total_tokens": 113438, + "reasoning_tokens": 4397 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 47.810582876205444, + "token_usage": { + "input_tokens": 60858, + "output_tokens": 4153, + "total_tokens": 65011, + "reasoning_tokens": 1874 + }, + "turn_count": 11 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 60.068111419677734, + "token_usage": { + "input_tokens": 84425, + "output_tokens": 4111, + "total_tokens": 88536, + "reasoning_tokens": 2115 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/employees__employee_performance_analysis.json b/model_results/grok-4-fast/employees__employee_performance_analysis.json new file mode 100644 index 00000000..1ac68d1a --- /dev/null +++ b/model_results/grok-4-fast/employees__employee_performance_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "employees__employee_performance_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 42.531134366989136, + "token_usage": { + "input_tokens": 101108, + "output_tokens": 4673, + "total_tokens": 105781, + "reasoning_tokens": 2128 + }, + "turn_count": 15 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 38.95880889892578, + "token_usage": { + "input_tokens": 73389, + "output_tokens": 2275, + "total_tokens": 75664, + "reasoning_tokens": 1028 + }, + "turn_count": 13 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 46.56830334663391, + "token_usage": { + "input_tokens": 77165, + "output_tokens": 3253, + "total_tokens": 80418, + "reasoning_tokens": 1903 + }, + "turn_count": 13 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 38.77199149131775, + "token_usage": { + "input_tokens": 71853, + "output_tokens": 3264, + "total_tokens": 75117, + "reasoning_tokens": 2311 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/employees__employee_project_tracking.json b/model_results/grok-4-fast/employees__employee_project_tracking.json new file mode 100644 index 00000000..447f4987 --- /dev/null +++ b/model_results/grok-4-fast/employees__employee_project_tracking.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "employees__employee_project_tracking", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 38.21620297431946, + "token_usage": { + "input_tokens": 105995, + "output_tokens": 4015, + "total_tokens": 110010, + "reasoning_tokens": 2170 + }, + "turn_count": 15 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 61.35165214538574, + "token_usage": { + "input_tokens": 189719, + "output_tokens": 5158, + "total_tokens": 194877, + "reasoning_tokens": 2441 + }, + "turn_count": 22 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 50.22589993476868, + "token_usage": { + "input_tokens": 126016, + "output_tokens": 4016, + "total_tokens": 130032, + "reasoning_tokens": 2054 + }, + "turn_count": 17 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 49.095847606658936, + "token_usage": { + "input_tokens": 137905, + "output_tokens": 4522, + "total_tokens": 142427, + "reasoning_tokens": 2495 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/employees__employee_retention_analysis.json b/model_results/grok-4-fast/employees__employee_retention_analysis.json new file mode 100644 index 00000000..d36a5765 --- /dev/null +++ b/model_results/grok-4-fast/employees__employee_retention_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "employees__employee_retention_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 112.12750005722046, + "token_usage": { + "input_tokens": 343929, + "output_tokens": 9799, + "total_tokens": 353728, + "reasoning_tokens": 6780 + }, + "turn_count": 30 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 71.78340077400208, + "token_usage": { + "input_tokens": 134433, + "output_tokens": 7488, + "total_tokens": 141921, + "reasoning_tokens": 5658 + }, + "turn_count": 16 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 45.265514612197876, + "token_usage": { + "input_tokens": 52759, + "output_tokens": 5291, + "total_tokens": 58050, + "reasoning_tokens": 3772 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 67.05810022354126, + "token_usage": { + "input_tokens": 109790, + "output_tokens": 5892, + "total_tokens": 115682, + "reasoning_tokens": 4339 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/employees__executive_dashboard_automation.json b/model_results/grok-4-fast/employees__executive_dashboard_automation.json new file mode 100644 index 00000000..2695b79c --- /dev/null +++ b/model_results/grok-4-fast/employees__executive_dashboard_automation.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "employees__executive_dashboard_automation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 112.6349561214447, + "token_usage": { + "input_tokens": 515137, + "output_tokens": 10512, + "total_tokens": 525649, + "reasoning_tokens": 6654 + }, + "turn_count": 39 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 74.55596232414246, + "token_usage": { + "input_tokens": 292539, + "output_tokens": 5974, + "total_tokens": 298513, + "reasoning_tokens": 3233 + }, + "turn_count": 28 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 129.5191683769226, + "token_usage": { + "input_tokens": 288811, + "output_tokens": 8704, + "total_tokens": 297515, + "reasoning_tokens": 5313 + }, + "turn_count": 25 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 141.3994083404541, + "token_usage": { + "input_tokens": 658363, + "output_tokens": 12204, + "total_tokens": 670567, + "reasoning_tokens": 7619 + }, + "turn_count": 41 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/employees__management_structure_analysis.json b/model_results/grok-4-fast/employees__management_structure_analysis.json new file mode 100644 index 00000000..d227c6b2 --- /dev/null +++ b/model_results/grok-4-fast/employees__management_structure_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "employees__management_structure_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 43.32529592514038, + "token_usage": { + "input_tokens": 100035, + "output_tokens": 5093, + "total_tokens": 105128, + "reasoning_tokens": 3571 + }, + "turn_count": 15 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 137.73046112060547, + "token_usage": { + "input_tokens": 483145, + "output_tokens": 10056, + "total_tokens": 493201, + "reasoning_tokens": 5341 + }, + "turn_count": 39 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 67.06036972999573, + "token_usage": { + "input_tokens": 174427, + "output_tokens": 5259, + "total_tokens": 179686, + "reasoning_tokens": 3405 + }, + "turn_count": 21 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 67.87331104278564, + "token_usage": { + "input_tokens": 134790, + "output_tokens": 7062, + "total_tokens": 141852, + "reasoning_tokens": 5300 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/eval_web__cloudflare_turnstile_challenge.json b/model_results/grok-4-fast/eval_web__cloudflare_turnstile_challenge.json new file mode 100644 index 00000000..c1a901ec --- /dev/null +++ b/model_results/grok-4-fast/eval_web__cloudflare_turnstile_challenge.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "eval_web__cloudflare_turnstile_challenge", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 145.92757034301758, + "token_usage": { + "input_tokens": 1779538, + "output_tokens": 5420, + "total_tokens": 1784958, + "reasoning_tokens": 4164 + }, + "turn_count": 34 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 74.85641479492188, + "token_usage": { + "input_tokens": 807838, + "output_tokens": 3620, + "total_tokens": 811458, + "reasoning_tokens": 2829 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 160.32092142105103, + "token_usage": { + "input_tokens": 1726538, + "output_tokens": 6651, + "total_tokens": 1733189, + "reasoning_tokens": 5075 + }, + "turn_count": 32 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 233.87894129753113, + "token_usage": { + "input_tokens": 2623119, + "output_tokens": 11261, + "total_tokens": 2634380, + "reasoning_tokens": 7880 + }, + "turn_count": 50 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/eval_web__extraction_table.json b/model_results/grok-4-fast/eval_web__extraction_table.json new file mode 100644 index 00000000..07068b09 --- /dev/null +++ b/model_results/grok-4-fast/eval_web__extraction_table.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "eval_web__extraction_table", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 75.35624742507935, + "token_usage": { + "input_tokens": 102036, + "output_tokens": 5233, + "total_tokens": 107269, + "reasoning_tokens": 2523 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 61.36948490142822, + "token_usage": { + "input_tokens": 76636, + "output_tokens": 5562, + "total_tokens": 82198, + "reasoning_tokens": 3888 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 25.2391197681427, + "token_usage": { + "input_tokens": 17839, + "output_tokens": 2672, + "total_tokens": 20511, + "reasoning_tokens": 2265 + }, + "turn_count": 3 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 58.82275152206421, + "token_usage": { + "input_tokens": 19945, + "output_tokens": 5074, + "total_tokens": 25019, + "reasoning_tokens": 3017 + }, + "turn_count": 3 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/file_context__duplicates_searching.json b/model_results/grok-4-fast/file_context__duplicates_searching.json new file mode 100644 index 00000000..b1c3ae16 --- /dev/null +++ b/model_results/grok-4-fast/file_context__duplicates_searching.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "file_context__duplicates_searching", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 31.07046866416931, + "token_usage": { + "input_tokens": 26848, + "output_tokens": 3670, + "total_tokens": 30518, + "reasoning_tokens": 1493 + }, + "turn_count": 6 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 26.182461500167847, + "token_usage": { + "input_tokens": 27086, + "output_tokens": 3681, + "total_tokens": 30767, + "reasoning_tokens": 1504 + }, + "turn_count": 6 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 19.116021394729614, + "token_usage": { + "input_tokens": 26537, + "output_tokens": 3409, + "total_tokens": 29946, + "reasoning_tokens": 1232 + }, + "turn_count": 6 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 24.408891916275024, + "token_usage": { + "input_tokens": 26791, + "output_tokens": 3480, + "total_tokens": 30271, + "reasoning_tokens": 1303 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/file_context__file_merging.json b/model_results/grok-4-fast/file_context__file_merging.json new file mode 100644 index 00000000..750651eb --- /dev/null +++ b/model_results/grok-4-fast/file_context__file_merging.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "file_context__file_merging", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 28.2689528465271, + "token_usage": { + "input_tokens": 20372, + "output_tokens": 2497, + "total_tokens": 22869, + "reasoning_tokens": 1649 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 16.43811583518982, + "token_usage": { + "input_tokens": 20052, + "output_tokens": 2070, + "total_tokens": 22122, + "reasoning_tokens": 1196 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 20.4749493598938, + "token_usage": { + "input_tokens": 16853, + "output_tokens": 2240, + "total_tokens": 19093, + "reasoning_tokens": 1416 + }, + "turn_count": 5 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 15.826157569885254, + "token_usage": { + "input_tokens": 16831, + "output_tokens": 2207, + "total_tokens": 19038, + "reasoning_tokens": 1462 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/file_context__file_splitting.json b/model_results/grok-4-fast/file_context__file_splitting.json new file mode 100644 index 00000000..c3df6107 --- /dev/null +++ b/model_results/grok-4-fast/file_context__file_splitting.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "file_context__file_splitting", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 100.13506531715393, + "token_usage": { + "input_tokens": 31606, + "output_tokens": 9275, + "total_tokens": 40881, + "reasoning_tokens": 8647 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 87.65612268447876, + "token_usage": { + "input_tokens": 28641, + "output_tokens": 13312, + "total_tokens": 41953, + "reasoning_tokens": 12417 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 78.489187002182, + "token_usage": { + "input_tokens": 17154, + "output_tokens": 7367, + "total_tokens": 24521, + "reasoning_tokens": 7131 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 115.47236323356628, + "token_usage": { + "input_tokens": 50185, + "output_tokens": 9795, + "total_tokens": 59980, + "reasoning_tokens": 8541 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/file_context__pattern_matching.json b/model_results/grok-4-fast/file_context__pattern_matching.json new file mode 100644 index 00000000..a98059b0 --- /dev/null +++ b/model_results/grok-4-fast/file_context__pattern_matching.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "file_context__pattern_matching", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 82.91513752937317, + "token_usage": { + "input_tokens": 29557, + "output_tokens": 7313, + "total_tokens": 36870, + "reasoning_tokens": 6449 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 58.620314598083496, + "token_usage": { + "input_tokens": 30052, + "output_tokens": 7533, + "total_tokens": 37585, + "reasoning_tokens": 6669 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 73.68318223953247, + "token_usage": { + "input_tokens": 30385, + "output_tokens": 8142, + "total_tokens": 38527, + "reasoning_tokens": 7278 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 63.82978868484497, + "token_usage": { + "input_tokens": 30232, + "output_tokens": 7012, + "total_tokens": 37244, + "reasoning_tokens": 6148 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/file_context__uppercase.json b/model_results/grok-4-fast/file_context__uppercase.json new file mode 100644 index 00000000..48242075 --- /dev/null +++ b/model_results/grok-4-fast/file_context__uppercase.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "file_context__uppercase", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 32.156017541885376, + "token_usage": { + "input_tokens": 30410, + "output_tokens": 4149, + "total_tokens": 34559, + "reasoning_tokens": 2543 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 21.8786678314209, + "token_usage": { + "input_tokens": 32069, + "output_tokens": 3201, + "total_tokens": 35270, + "reasoning_tokens": 1602 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 25.078603506088257, + "token_usage": { + "input_tokens": 21175, + "output_tokens": 3077, + "total_tokens": 24252, + "reasoning_tokens": 1463 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 32.34539723396301, + "token_usage": { + "input_tokens": 31591, + "output_tokens": 4366, + "total_tokens": 35957, + "reasoning_tokens": 2743 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/file_property__size_classification.json b/model_results/grok-4-fast/file_property__size_classification.json new file mode 100644 index 00000000..882ee7d7 --- /dev/null +++ b/model_results/grok-4-fast/file_property__size_classification.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "file_property__size_classification", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 25.813899993896484, + "token_usage": { + "input_tokens": 26934, + "output_tokens": 2779, + "total_tokens": 29713, + "reasoning_tokens": 1161 + }, + "turn_count": 7 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 18.65298867225647, + "token_usage": { + "input_tokens": 19742, + "output_tokens": 2351, + "total_tokens": 22093, + "reasoning_tokens": 1235 + }, + "turn_count": 6 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 16.838480234146118, + "token_usage": { + "input_tokens": 19408, + "output_tokens": 2060, + "total_tokens": 21468, + "reasoning_tokens": 921 + }, + "turn_count": 6 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 24.74107313156128, + "token_usage": { + "input_tokens": 31108, + "output_tokens": 2732, + "total_tokens": 33840, + "reasoning_tokens": 1064 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/file_property__time_classification.json b/model_results/grok-4-fast/file_property__time_classification.json new file mode 100644 index 00000000..945b4380 --- /dev/null +++ b/model_results/grok-4-fast/file_property__time_classification.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "file_property__time_classification", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 38.066346883773804, + "token_usage": { + "input_tokens": 58567, + "output_tokens": 6044, + "total_tokens": 64611, + "reasoning_tokens": 3672 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 58.31969094276428, + "token_usage": { + "input_tokens": 45709, + "output_tokens": 3605, + "total_tokens": 49314, + "reasoning_tokens": 2040 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 29.37819480895996, + "token_usage": { + "input_tokens": 46212, + "output_tokens": 3699, + "total_tokens": 49911, + "reasoning_tokens": 2134 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 29.231498956680298, + "token_usage": { + "input_tokens": 60163, + "output_tokens": 3964, + "total_tokens": 64127, + "reasoning_tokens": 2140 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/folder_structure__structure_analysis.json b/model_results/grok-4-fast/folder_structure__structure_analysis.json new file mode 100644 index 00000000..b202e94e --- /dev/null +++ b/model_results/grok-4-fast/folder_structure__structure_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "folder_structure__structure_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 124.99852323532104, + "token_usage": { + "input_tokens": 542336, + "output_tokens": 18742, + "total_tokens": 561078, + "reasoning_tokens": 17773 + }, + "turn_count": 17 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 216.39523315429688, + "token_usage": { + "input_tokens": 1586327, + "output_tokens": 24166, + "total_tokens": 1610493, + "reasoning_tokens": 18554 + }, + "turn_count": 67 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 178.13117289543152, + "token_usage": { + "input_tokens": 594382, + "output_tokens": 25546, + "total_tokens": 619928, + "reasoning_tokens": 18822 + }, + "turn_count": 25 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 175.06329870224, + "token_usage": { + "input_tokens": 439709, + "output_tokens": 20890, + "total_tokens": 460599, + "reasoning_tokens": 14363 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/folder_structure__structure_mirror.json b/model_results/grok-4-fast/folder_structure__structure_mirror.json new file mode 100644 index 00000000..8a64a335 --- /dev/null +++ b/model_results/grok-4-fast/folder_structure__structure_mirror.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "folder_structure__structure_mirror", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 58.23767924308777, + "token_usage": { + "input_tokens": 101245, + "output_tokens": 6458, + "total_tokens": 107703, + "reasoning_tokens": 4059 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 95.15689301490784, + "token_usage": { + "input_tokens": 483449, + "output_tokens": 6326, + "total_tokens": 489775, + "reasoning_tokens": 2935 + }, + "turn_count": 43 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 69.45335984230042, + "token_usage": { + "input_tokens": 146171, + "output_tokens": 8389, + "total_tokens": 154560, + "reasoning_tokens": 5453 + }, + "turn_count": 14 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 68.67535305023193, + "token_usage": { + "input_tokens": 120824, + "output_tokens": 8657, + "total_tokens": 129481, + "reasoning_tokens": 5892 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/harmony__fix_conflict.json b/model_results/grok-4-fast/harmony__fix_conflict.json new file mode 100644 index 00000000..bc04eb3f --- /dev/null +++ b/model_results/grok-4-fast/harmony__fix_conflict.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "harmony__fix_conflict", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 158.0662121772766, + "token_usage": { + "input_tokens": 1278048, + "output_tokens": 9931, + "total_tokens": 1287979, + "reasoning_tokens": 3607 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 132.37931084632874, + "token_usage": { + "input_tokens": 956874, + "output_tokens": 8651, + "total_tokens": 965525, + "reasoning_tokens": 2948 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 119.94980001449585, + "token_usage": { + "input_tokens": 1865531, + "output_tokens": 7148, + "total_tokens": 1872679, + "reasoning_tokens": 5271 + }, + "turn_count": 25 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 121.68680691719055, + "token_usage": { + "input_tokens": 1233935, + "output_tokens": 7964, + "total_tokens": 1241899, + "reasoning_tokens": 6046 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/harmony__issue_pr_commit_workflow.json b/model_results/grok-4-fast/harmony__issue_pr_commit_workflow.json new file mode 100644 index 00000000..5bbf5e0f --- /dev/null +++ b/model_results/grok-4-fast/harmony__issue_pr_commit_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "harmony__issue_pr_commit_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 57.13207173347473, + "token_usage": { + "input_tokens": 206076, + "output_tokens": 2795, + "total_tokens": 208871, + "reasoning_tokens": 1547 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 51.34369421005249, + "token_usage": { + "input_tokens": 186147, + "output_tokens": 3588, + "total_tokens": 189735, + "reasoning_tokens": 2096 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 50.264371395111084, + "token_usage": { + "input_tokens": 167917, + "output_tokens": 3125, + "total_tokens": 171042, + "reasoning_tokens": 1896 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 50.642306327819824, + "token_usage": { + "input_tokens": 229472, + "output_tokens": 3259, + "total_tokens": 232731, + "reasoning_tokens": 1684 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/harmony__issue_tagging_pr_closure.json b/model_results/grok-4-fast/harmony__issue_tagging_pr_closure.json new file mode 100644 index 00000000..6cacbe68 --- /dev/null +++ b/model_results/grok-4-fast/harmony__issue_tagging_pr_closure.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "harmony__issue_tagging_pr_closure", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 91.16628623008728, + "token_usage": { + "input_tokens": 445411, + "output_tokens": 6096, + "total_tokens": 451507, + "reasoning_tokens": 3738 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 69.33674907684326, + "token_usage": { + "input_tokens": 594909, + "output_tokens": 4877, + "total_tokens": 599786, + "reasoning_tokens": 3163 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 185.30562829971313, + "token_usage": { + "input_tokens": 247766, + "output_tokens": 16727, + "total_tokens": 264493, + "reasoning_tokens": 4748 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 149.45539236068726, + "token_usage": { + "input_tokens": 586078, + "output_tokens": 9698, + "total_tokens": 595776, + "reasoning_tokens": 6097 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/harmony__multi_branch_commit_aggregation.json b/model_results/grok-4-fast/harmony__multi_branch_commit_aggregation.json new file mode 100644 index 00000000..76fcf34c --- /dev/null +++ b/model_results/grok-4-fast/harmony__multi_branch_commit_aggregation.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "harmony__multi_branch_commit_aggregation", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 105.11098074913025, + "token_usage": { + "input_tokens": 345988, + "output_tokens": 10157, + "total_tokens": 356145, + "reasoning_tokens": 6669 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 84.2107036113739, + "token_usage": { + "input_tokens": 535977, + "output_tokens": 7529, + "total_tokens": 543506, + "reasoning_tokens": 4897 + }, + "turn_count": 12 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 85.45590472221375, + "token_usage": { + "input_tokens": 180472, + "output_tokens": 9162, + "total_tokens": 189634, + "reasoning_tokens": 6482 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 78.71429514884949, + "token_usage": { + "input_tokens": 215336, + "output_tokens": 8242, + "total_tokens": 223578, + "reasoning_tokens": 5688 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/harmony__release_management_workflow.json b/model_results/grok-4-fast/harmony__release_management_workflow.json new file mode 100644 index 00000000..edfa6088 --- /dev/null +++ b/model_results/grok-4-fast/harmony__release_management_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "harmony__release_management_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 139.07860112190247, + "token_usage": { + "input_tokens": 1482872, + "output_tokens": 9260, + "total_tokens": 1492132, + "reasoning_tokens": 5998 + }, + "turn_count": 32 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 121.97672057151794, + "token_usage": { + "input_tokens": 1221456, + "output_tokens": 8736, + "total_tokens": 1230192, + "reasoning_tokens": 6201 + }, + "turn_count": 27 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 50.831056118011475, + "token_usage": { + "input_tokens": 617227, + "output_tokens": 3417, + "total_tokens": 620644, + "reasoning_tokens": 1978 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 97.53643751144409, + "token_usage": { + "input_tokens": 737833, + "output_tokens": 7788, + "total_tokens": 745621, + "reasoning_tokens": 4724 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/it_trouble_shooting_hub__asset_retirement_migration.json b/model_results/grok-4-fast/it_trouble_shooting_hub__asset_retirement_migration.json new file mode 100644 index 00000000..63f028a5 --- /dev/null +++ b/model_results/grok-4-fast/it_trouble_shooting_hub__asset_retirement_migration.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "it_trouble_shooting_hub__asset_retirement_migration", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 87.78935718536377, + "token_usage": { + "input_tokens": 277371, + "output_tokens": 6430, + "total_tokens": 283801, + "reasoning_tokens": 4535 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 49.56605386734009, + "token_usage": { + "input_tokens": 187634, + "output_tokens": 5542, + "total_tokens": 193176, + "reasoning_tokens": 4157 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 152.46633458137512, + "token_usage": { + "input_tokens": 902256, + "output_tokens": 11601, + "total_tokens": 913857, + "reasoning_tokens": 7763 + }, + "turn_count": 45 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 91.61820769309998, + "token_usage": { + "input_tokens": 507113, + "output_tokens": 8418, + "total_tokens": 515531, + "reasoning_tokens": 6454 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/it_trouble_shooting_hub__security_audit_ticket.json b/model_results/grok-4-fast/it_trouble_shooting_hub__security_audit_ticket.json new file mode 100644 index 00000000..50f93f5c --- /dev/null +++ b/model_results/grok-4-fast/it_trouble_shooting_hub__security_audit_ticket.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "it_trouble_shooting_hub__security_audit_ticket", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 73.72738265991211, + "token_usage": { + "input_tokens": 298650, + "output_tokens": 6796, + "total_tokens": 305446, + "reasoning_tokens": 5373 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 63.83225989341736, + "token_usage": { + "input_tokens": 262008, + "output_tokens": 5468, + "total_tokens": 267476, + "reasoning_tokens": 4027 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 47.03505611419678, + "token_usage": { + "input_tokens": 134029, + "output_tokens": 4678, + "total_tokens": 138707, + "reasoning_tokens": 3113 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 66.14121913909912, + "token_usage": { + "input_tokens": 167096, + "output_tokens": 4613, + "total_tokens": 171709, + "reasoning_tokens": 3549 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/it_trouble_shooting_hub__verification_expired_update.json b/model_results/grok-4-fast/it_trouble_shooting_hub__verification_expired_update.json new file mode 100644 index 00000000..4b1abb5e --- /dev/null +++ b/model_results/grok-4-fast/it_trouble_shooting_hub__verification_expired_update.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "it_trouble_shooting_hub__verification_expired_update", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 91.70930600166321, + "token_usage": { + "input_tokens": 385358, + "output_tokens": 9969, + "total_tokens": 395327, + "reasoning_tokens": 8523 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 80.19039297103882, + "token_usage": { + "input_tokens": 450985, + "output_tokens": 6961, + "total_tokens": 457946, + "reasoning_tokens": 5075 + }, + "turn_count": 27 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 109.55777955055237, + "token_usage": { + "input_tokens": 632648, + "output_tokens": 7090, + "total_tokens": 639738, + "reasoning_tokens": 5443 + }, + "turn_count": 23 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 144.57292103767395, + "token_usage": { + "input_tokens": 774930, + "output_tokens": 10179, + "total_tokens": 785109, + "reasoning_tokens": 7214 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/japan_travel_planner__daily_itinerary_overview.json b/model_results/grok-4-fast/japan_travel_planner__daily_itinerary_overview.json new file mode 100644 index 00000000..9bb0acef --- /dev/null +++ b/model_results/grok-4-fast/japan_travel_planner__daily_itinerary_overview.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "japan_travel_planner__daily_itinerary_overview", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 46.992114782333374, + "token_usage": { + "input_tokens": 319155, + "output_tokens": 3785, + "total_tokens": 322940, + "reasoning_tokens": 3087 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 373.01144433021545, + "token_usage": { + "input_tokens": 144028, + "output_tokens": 71698, + "total_tokens": 215726, + "reasoning_tokens": 2743 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 47.8009614944458, + "token_usage": { + "input_tokens": 219979, + "output_tokens": 6032, + "total_tokens": 226011, + "reasoning_tokens": 4680 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 123.42545127868652, + "token_usage": { + "input_tokens": 912289, + "output_tokens": 13390, + "total_tokens": 925679, + "reasoning_tokens": 5512 + }, + "turn_count": 16 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/japan_travel_planner__packing_progress_summary.json b/model_results/grok-4-fast/japan_travel_planner__packing_progress_summary.json new file mode 100644 index 00000000..90a5f834 --- /dev/null +++ b/model_results/grok-4-fast/japan_travel_planner__packing_progress_summary.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "japan_travel_planner__packing_progress_summary", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 115.71496367454529, + "token_usage": { + "input_tokens": 556806, + "output_tokens": 9627, + "total_tokens": 566433, + "reasoning_tokens": 4918 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 175.28860759735107, + "token_usage": { + "input_tokens": 353562, + "output_tokens": 18224, + "total_tokens": 371786, + "reasoning_tokens": 8794 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 454.695285320282, + "token_usage": { + "input_tokens": 13886094, + "output_tokens": 14226, + "total_tokens": 13900320, + "reasoning_tokens": 11503 + }, + "turn_count": 36 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 65.58109784126282, + "token_usage": { + "input_tokens": 367626, + "output_tokens": 6645, + "total_tokens": 374271, + "reasoning_tokens": 2888 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/japan_travel_planner__remove_osaka_itinerary.json b/model_results/grok-4-fast/japan_travel_planner__remove_osaka_itinerary.json new file mode 100644 index 00000000..8fadc095 --- /dev/null +++ b/model_results/grok-4-fast/japan_travel_planner__remove_osaka_itinerary.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "japan_travel_planner__remove_osaka_itinerary", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 26.992112398147583, + "token_usage": { + "input_tokens": 53136, + "output_tokens": 1918, + "total_tokens": 55054, + "reasoning_tokens": 1549 + }, + "turn_count": 4 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 28.564209699630737, + "token_usage": { + "input_tokens": 104649, + "output_tokens": 2152, + "total_tokens": 106801, + "reasoning_tokens": 1563 + }, + "turn_count": 8 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 42.16823625564575, + "token_usage": { + "input_tokens": 188289, + "output_tokens": 2700, + "total_tokens": 190989, + "reasoning_tokens": 2231 + }, + "turn_count": 7 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 24.776986122131348, + "token_usage": { + "input_tokens": 67944, + "output_tokens": 1521, + "total_tokens": 69465, + "reasoning_tokens": 1077 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/japan_travel_planner__restaurant_expenses_sync.json b/model_results/grok-4-fast/japan_travel_planner__restaurant_expenses_sync.json new file mode 100644 index 00000000..8c403a08 --- /dev/null +++ b/model_results/grok-4-fast/japan_travel_planner__restaurant_expenses_sync.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "japan_travel_planner__restaurant_expenses_sync", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 62.56175231933594, + "token_usage": { + "input_tokens": 266025, + "output_tokens": 5208, + "total_tokens": 271233, + "reasoning_tokens": 3514 + }, + "turn_count": 20 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 39.90877628326416, + "token_usage": { + "input_tokens": 108145, + "output_tokens": 5346, + "total_tokens": 113491, + "reasoning_tokens": 3407 + }, + "turn_count": 9 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 55.826473236083984, + "token_usage": { + "input_tokens": 247098, + "output_tokens": 4732, + "total_tokens": 251830, + "reasoning_tokens": 3392 + }, + "turn_count": 15 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 82.0383071899414, + "token_usage": { + "input_tokens": 375514, + "output_tokens": 6312, + "total_tokens": 381826, + "reasoning_tokens": 3977 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/legal_document__dispute_review.json b/model_results/grok-4-fast/legal_document__dispute_review.json new file mode 100644 index 00000000..a920fe02 --- /dev/null +++ b/model_results/grok-4-fast/legal_document__dispute_review.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "legal_document__dispute_review", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 23.48933219909668, + "token_usage": { + "input_tokens": 137806, + "output_tokens": 1871, + "total_tokens": 139677, + "reasoning_tokens": 1425 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 29.011127710342407, + "token_usage": { + "input_tokens": 201190, + "output_tokens": 2389, + "total_tokens": 203579, + "reasoning_tokens": 1904 + }, + "turn_count": 8 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 24.237136840820312, + "token_usage": { + "input_tokens": 136765, + "output_tokens": 2726, + "total_tokens": 139491, + "reasoning_tokens": 2336 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 37.84913921356201, + "token_usage": { + "input_tokens": 151078, + "output_tokens": 3277, + "total_tokens": 154355, + "reasoning_tokens": 2772 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/legal_document__individual_comments.json b/model_results/grok-4-fast/legal_document__individual_comments.json new file mode 100644 index 00000000..d935cfa6 --- /dev/null +++ b/model_results/grok-4-fast/legal_document__individual_comments.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "legal_document__individual_comments", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 36.647098779678345, + "token_usage": { + "input_tokens": 303404, + "output_tokens": 3149, + "total_tokens": 306553, + "reasoning_tokens": 2614 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 38.81229066848755, + "token_usage": { + "input_tokens": 389333, + "output_tokens": 3489, + "total_tokens": 392822, + "reasoning_tokens": 2716 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 31.852614641189575, + "token_usage": { + "input_tokens": 180734, + "output_tokens": 3287, + "total_tokens": 184021, + "reasoning_tokens": 2736 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 40.06143379211426, + "token_usage": { + "input_tokens": 306461, + "output_tokens": 3161, + "total_tokens": 309622, + "reasoning_tokens": 2599 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/legal_document__solution_tracing.json b/model_results/grok-4-fast/legal_document__solution_tracing.json new file mode 100644 index 00000000..ba4f0b39 --- /dev/null +++ b/model_results/grok-4-fast/legal_document__solution_tracing.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "legal_document__solution_tracing", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 49.13685917854309, + "token_usage": { + "input_tokens": 379391, + "output_tokens": 3958, + "total_tokens": 383349, + "reasoning_tokens": 3339 + }, + "turn_count": 8 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 49.26004719734192, + "token_usage": { + "input_tokens": 378323, + "output_tokens": 4112, + "total_tokens": 382435, + "reasoning_tokens": 3491 + }, + "turn_count": 8 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 42.50874185562134, + "token_usage": { + "input_tokens": 402227, + "output_tokens": 4112, + "total_tokens": 406339, + "reasoning_tokens": 3471 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 43.66520857810974, + "token_usage": { + "input_tokens": 404606, + "output_tokens": 4548, + "total_tokens": 409154, + "reasoning_tokens": 3811 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/lego__consistency_enforcement.json b/model_results/grok-4-fast/lego__consistency_enforcement.json new file mode 100644 index 00000000..90efc334 --- /dev/null +++ b/model_results/grok-4-fast/lego__consistency_enforcement.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "lego__consistency_enforcement", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 68.30837321281433, + "token_usage": { + "input_tokens": 481345, + "output_tokens": 5469, + "total_tokens": 486814, + "reasoning_tokens": 3546 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 104.10069179534912, + "token_usage": { + "input_tokens": 451890, + "output_tokens": 8675, + "total_tokens": 460565, + "reasoning_tokens": 6453 + }, + "turn_count": 17 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 79.97889614105225, + "token_usage": { + "input_tokens": 276928, + "output_tokens": 6839, + "total_tokens": 283767, + "reasoning_tokens": 3607 + }, + "turn_count": 13 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 69.09242391586304, + "token_usage": { + "input_tokens": 430884, + "output_tokens": 6994, + "total_tokens": 437878, + "reasoning_tokens": 3735 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/lego__database_security_policies.json b/model_results/grok-4-fast/lego__database_security_policies.json new file mode 100644 index 00000000..c653acaa --- /dev/null +++ b/model_results/grok-4-fast/lego__database_security_policies.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "lego__database_security_policies", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 52.30773138999939, + "token_usage": { + "input_tokens": 165312, + "output_tokens": 5291, + "total_tokens": 170603, + "reasoning_tokens": 1810 + }, + "turn_count": 24 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 83.63074851036072, + "token_usage": { + "input_tokens": 191979, + "output_tokens": 6961, + "total_tokens": 198940, + "reasoning_tokens": 2940 + }, + "turn_count": 26 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 74.1859028339386, + "token_usage": { + "input_tokens": 205754, + "output_tokens": 5882, + "total_tokens": 211636, + "reasoning_tokens": 2024 + }, + "turn_count": 28 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 74.46791768074036, + "token_usage": { + "input_tokens": 258020, + "output_tokens": 6717, + "total_tokens": 264737, + "reasoning_tokens": 2612 + }, + "turn_count": 32 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/lego__transactional_inventory_transfer.json b/model_results/grok-4-fast/lego__transactional_inventory_transfer.json new file mode 100644 index 00000000..75961640 --- /dev/null +++ b/model_results/grok-4-fast/lego__transactional_inventory_transfer.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "lego__transactional_inventory_transfer", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 117.18111109733582, + "token_usage": { + "input_tokens": 115872, + "output_tokens": 15654, + "total_tokens": 131526, + "reasoning_tokens": 13751 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 164.9237825870514, + "token_usage": { + "input_tokens": 505122, + "output_tokens": 13027, + "total_tokens": 518149, + "reasoning_tokens": 9360 + }, + "turn_count": 34 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 96.16848468780518, + "token_usage": { + "input_tokens": 36427, + "output_tokens": 8141, + "total_tokens": 44568, + "reasoning_tokens": 6385 + }, + "turn_count": 6 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 63.01567244529724, + "token_usage": { + "input_tokens": 64635, + "output_tokens": 8024, + "total_tokens": 72659, + "reasoning_tokens": 4248 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/mcpmark-cicd__deployment_status_workflow.json b/model_results/grok-4-fast/mcpmark-cicd__deployment_status_workflow.json new file mode 100644 index 00000000..bde52f45 --- /dev/null +++ b/model_results/grok-4-fast/mcpmark-cicd__deployment_status_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "mcpmark-cicd__deployment_status_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 57.17888879776001, + "token_usage": { + "input_tokens": 86080, + "output_tokens": 4983, + "total_tokens": 91063, + "reasoning_tokens": 2856 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 37.55769395828247, + "token_usage": { + "input_tokens": 84602, + "output_tokens": 4491, + "total_tokens": 89093, + "reasoning_tokens": 2609 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 58.69420385360718, + "token_usage": { + "input_tokens": 93903, + "output_tokens": 6910, + "total_tokens": 100813, + "reasoning_tokens": 4900 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 69.0181930065155, + "token_usage": { + "input_tokens": 87421, + "output_tokens": 5443, + "total_tokens": 92864, + "reasoning_tokens": 3386 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/mcpmark-cicd__issue_management_workflow.json b/model_results/grok-4-fast/mcpmark-cicd__issue_management_workflow.json new file mode 100644 index 00000000..b308f76f --- /dev/null +++ b/model_results/grok-4-fast/mcpmark-cicd__issue_management_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "mcpmark-cicd__issue_management_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 81.23488736152649, + "token_usage": { + "input_tokens": 232946, + "output_tokens": 8666, + "total_tokens": 241612, + "reasoning_tokens": 4841 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 99.36858129501343, + "token_usage": { + "input_tokens": 261304, + "output_tokens": 9938, + "total_tokens": 271242, + "reasoning_tokens": 6376 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 82.35587882995605, + "token_usage": { + "input_tokens": 166010, + "output_tokens": 11508, + "total_tokens": 177518, + "reasoning_tokens": 7859 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 82.02052307128906, + "token_usage": { + "input_tokens": 254939, + "output_tokens": 10525, + "total_tokens": 265464, + "reasoning_tokens": 6807 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/mcpmark-cicd__linting_ci_workflow.json b/model_results/grok-4-fast/mcpmark-cicd__linting_ci_workflow.json new file mode 100644 index 00000000..24f9ef77 --- /dev/null +++ b/model_results/grok-4-fast/mcpmark-cicd__linting_ci_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "mcpmark-cicd__linting_ci_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 39.48687195777893, + "token_usage": { + "input_tokens": 93496, + "output_tokens": 2789, + "total_tokens": 96285, + "reasoning_tokens": 1867 + }, + "turn_count": 6 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 33.43517184257507, + "token_usage": { + "input_tokens": 90667, + "output_tokens": 2464, + "total_tokens": 93131, + "reasoning_tokens": 1554 + }, + "turn_count": 6 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 39.33463096618652, + "token_usage": { + "input_tokens": 78225, + "output_tokens": 2565, + "total_tokens": 80790, + "reasoning_tokens": 1676 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 38.12186312675476, + "token_usage": { + "input_tokens": 78418, + "output_tokens": 2642, + "total_tokens": 81060, + "reasoning_tokens": 1701 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/mcpmark-cicd__pr_automation_workflow.json b/model_results/grok-4-fast/mcpmark-cicd__pr_automation_workflow.json new file mode 100644 index 00000000..6929f4cc --- /dev/null +++ b/model_results/grok-4-fast/mcpmark-cicd__pr_automation_workflow.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "mcpmark-cicd__pr_automation_workflow", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 31.984736680984497, + "token_usage": { + "input_tokens": 81540, + "output_tokens": 3720, + "total_tokens": 85260, + "reasoning_tokens": 2078 + }, + "turn_count": 5 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 64.02203607559204, + "token_usage": { + "input_tokens": 80728, + "output_tokens": 4008, + "total_tokens": 84736, + "reasoning_tokens": 2497 + }, + "turn_count": 5 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 52.280176639556885, + "token_usage": { + "input_tokens": 81378, + "output_tokens": 3641, + "total_tokens": 85019, + "reasoning_tokens": 1916 + }, + "turn_count": 5 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 56.474762201309204, + "token_usage": { + "input_tokens": 81605, + "output_tokens": 3646, + "total_tokens": 85251, + "reasoning_tokens": 2448 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/missing-semester__assign_contributor_labels.json b/model_results/grok-4-fast/missing-semester__assign_contributor_labels.json new file mode 100644 index 00000000..88518dc7 --- /dev/null +++ b/model_results/grok-4-fast/missing-semester__assign_contributor_labels.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "missing-semester__assign_contributor_labels", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 55.51180863380432, + "token_usage": { + "input_tokens": 165074, + "output_tokens": 4050, + "total_tokens": 169124, + "reasoning_tokens": 2929 + }, + "turn_count": 7 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 48.97187948226929, + "token_usage": { + "input_tokens": 150499, + "output_tokens": 4446, + "total_tokens": 154945, + "reasoning_tokens": 3891 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 69.35130429267883, + "token_usage": { + "input_tokens": 188252, + "output_tokens": 5373, + "total_tokens": 193625, + "reasoning_tokens": 3764 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 64.27985310554504, + "token_usage": { + "input_tokens": 217762, + "output_tokens": 4888, + "total_tokens": 222650, + "reasoning_tokens": 3376 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/missing-semester__find_legacy_name.json b/model_results/grok-4-fast/missing-semester__find_legacy_name.json new file mode 100644 index 00000000..8c4fac16 --- /dev/null +++ b/model_results/grok-4-fast/missing-semester__find_legacy_name.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "missing-semester__find_legacy_name", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 97.89311742782593, + "token_usage": { + "input_tokens": 263734, + "output_tokens": 8011, + "total_tokens": 271745, + "reasoning_tokens": 2588 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 72.64546275138855, + "token_usage": { + "input_tokens": 123187, + "output_tokens": 6000, + "total_tokens": 129187, + "reasoning_tokens": 5639 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 122.08328175544739, + "token_usage": { + "input_tokens": 719413, + "output_tokens": 8831, + "total_tokens": 728244, + "reasoning_tokens": 5668 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 112.49965167045593, + "token_usage": { + "input_tokens": 633035, + "output_tokens": 7615, + "total_tokens": 640650, + "reasoning_tokens": 3994 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/missing-semester__find_salient_file.json b/model_results/grok-4-fast/missing-semester__find_salient_file.json new file mode 100644 index 00000000..8c02604a --- /dev/null +++ b/model_results/grok-4-fast/missing-semester__find_salient_file.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "github", + "task": "missing-semester__find_salient_file", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 114.99301266670227, + "token_usage": { + "input_tokens": 1085121, + "output_tokens": 7416, + "total_tokens": 1092537, + "reasoning_tokens": 5868 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 85.67410707473755, + "token_usage": { + "input_tokens": 360696, + "output_tokens": 6896, + "total_tokens": 367592, + "reasoning_tokens": 4442 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 108.95954179763794, + "token_usage": { + "input_tokens": 425883, + "output_tokens": 7102, + "total_tokens": 432985, + "reasoning_tokens": 4500 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 38.620139837265015, + "token_usage": { + "input_tokens": 108435, + "output_tokens": 3056, + "total_tokens": 111491, + "reasoning_tokens": 2599 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/online_resume__layout_adjustment.json b/model_results/grok-4-fast/online_resume__layout_adjustment.json new file mode 100644 index 00000000..761c8ee9 --- /dev/null +++ b/model_results/grok-4-fast/online_resume__layout_adjustment.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "online_resume__layout_adjustment", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 97.22220754623413, + "token_usage": { + "input_tokens": 436107, + "output_tokens": 8564, + "total_tokens": 444671, + "reasoning_tokens": 5824 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 72.72354555130005, + "token_usage": { + "input_tokens": 466141, + "output_tokens": 6985, + "total_tokens": 473126, + "reasoning_tokens": 5731 + }, + "turn_count": 20 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 13.191792964935303, + "token_usage": { + "input_tokens": 18362, + "output_tokens": 1037, + "total_tokens": 19399, + "reasoning_tokens": 921 + }, + "turn_count": 4 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 148.71358561515808, + "token_usage": { + "input_tokens": 1547591, + "output_tokens": 7623, + "total_tokens": 1555214, + "reasoning_tokens": 5551 + }, + "turn_count": 36 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/online_resume__projects_section_update.json b/model_results/grok-4-fast/online_resume__projects_section_update.json new file mode 100644 index 00000000..34dd286e --- /dev/null +++ b/model_results/grok-4-fast/online_resume__projects_section_update.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "online_resume__projects_section_update", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 100.46130013465881, + "token_usage": { + "input_tokens": 602996, + "output_tokens": 7302, + "total_tokens": 610298, + "reasoning_tokens": 5272 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 97.22893905639648, + "token_usage": { + "input_tokens": 768191, + "output_tokens": 7471, + "total_tokens": 775662, + "reasoning_tokens": 5080 + }, + "turn_count": 32 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 64.34393429756165, + "token_usage": { + "input_tokens": 283142, + "output_tokens": 5006, + "total_tokens": 288148, + "reasoning_tokens": 3752 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 110.76311779022217, + "token_usage": { + "input_tokens": 520703, + "output_tokens": 7341, + "total_tokens": 528044, + "reasoning_tokens": 4877 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/online_resume__skills_development_tracker.json b/model_results/grok-4-fast/online_resume__skills_development_tracker.json new file mode 100644 index 00000000..1e304e5d --- /dev/null +++ b/model_results/grok-4-fast/online_resume__skills_development_tracker.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "online_resume__skills_development_tracker", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 148.39083409309387, + "token_usage": { + "input_tokens": 606963, + "output_tokens": 11980, + "total_tokens": 618943, + "reasoning_tokens": 8702 + }, + "turn_count": 33 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 92.16241908073425, + "token_usage": { + "input_tokens": 216474, + "output_tokens": 9529, + "total_tokens": 226003, + "reasoning_tokens": 7334 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 103.72020506858826, + "token_usage": { + "input_tokens": 415296, + "output_tokens": 9315, + "total_tokens": 424611, + "reasoning_tokens": 7033 + }, + "turn_count": 22 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 41.46541929244995, + "token_usage": { + "input_tokens": 22037, + "output_tokens": 3834, + "total_tokens": 25871, + "reasoning_tokens": 2454 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/online_resume__work_history_addition.json b/model_results/grok-4-fast/online_resume__work_history_addition.json new file mode 100644 index 00000000..8d3aad72 --- /dev/null +++ b/model_results/grok-4-fast/online_resume__work_history_addition.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "online_resume__work_history_addition", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 105.77533602714539, + "token_usage": { + "input_tokens": 404023, + "output_tokens": 10149, + "total_tokens": 414172, + "reasoning_tokens": 7596 + }, + "turn_count": 21 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 95.72051048278809, + "token_usage": { + "input_tokens": 464662, + "output_tokens": 8073, + "total_tokens": 472735, + "reasoning_tokens": 6327 + }, + "turn_count": 25 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 27.975849151611328, + "token_usage": { + "input_tokens": 43906, + "output_tokens": 2619, + "total_tokens": 46525, + "reasoning_tokens": 2352 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 132.7514307498932, + "token_usage": { + "input_tokens": 385557, + "output_tokens": 6907, + "total_tokens": 392464, + "reasoning_tokens": 5496 + }, + "turn_count": 21 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/papers__author_folders.json b/model_results/grok-4-fast/papers__author_folders.json new file mode 100644 index 00000000..172bb6e2 --- /dev/null +++ b/model_results/grok-4-fast/papers__author_folders.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "papers__author_folders", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 412.186758518219, + "token_usage": { + "input_tokens": 5831585, + "output_tokens": 37565, + "total_tokens": 5869150, + "reasoning_tokens": 6750 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 98.62545466423035, + "token_usage": { + "input_tokens": 809332, + "output_tokens": 8520, + "total_tokens": 817852, + "reasoning_tokens": 7183 + }, + "turn_count": 7 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 830.5420525074005, + "token_usage": { + "input_tokens": 21332562, + "output_tokens": 10848, + "total_tokens": 21343410, + "reasoning_tokens": 7027 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 221.21445202827454, + "token_usage": { + "input_tokens": 2120484, + "output_tokens": 9674, + "total_tokens": 2130158, + "reasoning_tokens": 5969 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/papers__find_math_paper.json b/model_results/grok-4-fast/papers__find_math_paper.json new file mode 100644 index 00000000..b611fd4e --- /dev/null +++ b/model_results/grok-4-fast/papers__find_math_paper.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "papers__find_math_paper", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 29.68813705444336, + "token_usage": { + "input_tokens": 106082, + "output_tokens": 2087, + "total_tokens": 108169, + "reasoning_tokens": 1586 + }, + "turn_count": 9 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 25.301192045211792, + "token_usage": { + "input_tokens": 93055, + "output_tokens": 2903, + "total_tokens": 95958, + "reasoning_tokens": 2471 + }, + "turn_count": 8 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 206.4177656173706, + "token_usage": { + "input_tokens": 91771, + "output_tokens": 5099, + "total_tokens": 96870, + "reasoning_tokens": 4184 + }, + "turn_count": 7 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 22.769134283065796, + "token_usage": { + "input_tokens": 96312, + "output_tokens": 3170, + "total_tokens": 99482, + "reasoning_tokens": 2738 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/papers__organize_legacy_papers.json b/model_results/grok-4-fast/papers__organize_legacy_papers.json new file mode 100644 index 00000000..a868188f --- /dev/null +++ b/model_results/grok-4-fast/papers__organize_legacy_papers.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "papers__organize_legacy_papers", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 737.5350813865662, + "token_usage": { + "input_tokens": 1651267, + "output_tokens": 7636, + "total_tokens": 1658903, + "reasoning_tokens": 3846 + }, + "turn_count": 35 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 68.53865361213684, + "token_usage": { + "input_tokens": 1402969, + "output_tokens": 6299, + "total_tokens": 1409268, + "reasoning_tokens": 3152 + }, + "turn_count": 18 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 99.52957820892334, + "token_usage": { + "input_tokens": 1867993, + "output_tokens": 9540, + "total_tokens": 1877533, + "reasoning_tokens": 5838 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 235.27552676200867, + "token_usage": { + "input_tokens": 7632065, + "output_tokens": 11999, + "total_tokens": 7644064, + "reasoning_tokens": 7876 + }, + "turn_count": 57 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/python_roadmap__expert_level_lessons.json b/model_results/grok-4-fast/python_roadmap__expert_level_lessons.json new file mode 100644 index 00000000..92520610 --- /dev/null +++ b/model_results/grok-4-fast/python_roadmap__expert_level_lessons.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "python_roadmap__expert_level_lessons", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 92.83392596244812, + "token_usage": { + "input_tokens": 392706, + "output_tokens": 6863, + "total_tokens": 399569, + "reasoning_tokens": 4876 + }, + "turn_count": 27 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 94.1509051322937, + "token_usage": { + "input_tokens": 361773, + "output_tokens": 7281, + "total_tokens": 369054, + "reasoning_tokens": 5634 + }, + "turn_count": 25 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 104.47433710098267, + "token_usage": { + "input_tokens": 174030, + "output_tokens": 10573, + "total_tokens": 184603, + "reasoning_tokens": 8293 + }, + "turn_count": 7 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 118.02387833595276, + "token_usage": { + "input_tokens": 553560, + "output_tokens": 5849, + "total_tokens": 559409, + "reasoning_tokens": 4391 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/python_roadmap__learning_metrics_dashboard.json b/model_results/grok-4-fast/python_roadmap__learning_metrics_dashboard.json new file mode 100644 index 00000000..960ba3ce --- /dev/null +++ b/model_results/grok-4-fast/python_roadmap__learning_metrics_dashboard.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "python_roadmap__learning_metrics_dashboard", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 546.8645763397217, + "token_usage": { + "input_tokens": 385843, + "output_tokens": 75802, + "total_tokens": 461645, + "reasoning_tokens": 3986 + }, + "turn_count": 12 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 71.43365120887756, + "token_usage": { + "input_tokens": 678601, + "output_tokens": 6206, + "total_tokens": 684807, + "reasoning_tokens": 4491 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 74.8179440498352, + "token_usage": { + "input_tokens": 389052, + "output_tokens": 6685, + "total_tokens": 395737, + "reasoning_tokens": 5351 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 135.74409294128418, + "token_usage": { + "input_tokens": 1456294, + "output_tokens": 9547, + "total_tokens": 1465841, + "reasoning_tokens": 6803 + }, + "turn_count": 32 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/reddit__ai_data_analyst.json b/model_results/grok-4-fast/reddit__ai_data_analyst.json new file mode 100644 index 00000000..b6b029d3 --- /dev/null +++ b/model_results/grok-4-fast/reddit__ai_data_analyst.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "reddit__ai_data_analyst", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 108.78989100456238, + "token_usage": { + "input_tokens": 1372601, + "output_tokens": 4798, + "total_tokens": 1377399, + "reasoning_tokens": 3510 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 128.2094602584839, + "token_usage": { + "input_tokens": 1848277, + "output_tokens": 7176, + "total_tokens": 1855453, + "reasoning_tokens": 6042 + }, + "turn_count": 21 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 91.6964066028595, + "token_usage": { + "input_tokens": 1634327, + "output_tokens": 4762, + "total_tokens": 1639089, + "reasoning_tokens": 3781 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 113.4655659198761, + "token_usage": { + "input_tokens": 1975478, + "output_tokens": 3899, + "total_tokens": 1979377, + "reasoning_tokens": 3028 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/reddit__budget_europe_travel.json b/model_results/grok-4-fast/reddit__budget_europe_travel.json new file mode 100644 index 00000000..ad9e974d --- /dev/null +++ b/model_results/grok-4-fast/reddit__budget_europe_travel.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "reddit__budget_europe_travel", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 68.495689868927, + "token_usage": { + "input_tokens": 560735, + "output_tokens": 3500, + "total_tokens": 564235, + "reasoning_tokens": 2246 + }, + "turn_count": 22 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 75.1107017993927, + "token_usage": { + "input_tokens": 492329, + "output_tokens": 3031, + "total_tokens": 495360, + "reasoning_tokens": 1769 + }, + "turn_count": 19 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 74.9792993068695, + "token_usage": { + "input_tokens": 505275, + "output_tokens": 3250, + "total_tokens": 508525, + "reasoning_tokens": 1764 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 72.11851787567139, + "token_usage": { + "input_tokens": 494449, + "output_tokens": 3332, + "total_tokens": 497781, + "reasoning_tokens": 2074 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/reddit__buyitforlife_research.json b/model_results/grok-4-fast/reddit__buyitforlife_research.json new file mode 100644 index 00000000..55f785da --- /dev/null +++ b/model_results/grok-4-fast/reddit__buyitforlife_research.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "reddit__buyitforlife_research", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 57.46671414375305, + "token_usage": { + "input_tokens": 443162, + "output_tokens": 2287, + "total_tokens": 445449, + "reasoning_tokens": 1608 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 68.92222189903259, + "token_usage": { + "input_tokens": 1048576, + "output_tokens": 3266, + "total_tokens": 1051842, + "reasoning_tokens": 2440 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 61.987855672836304, + "token_usage": { + "input_tokens": 731634, + "output_tokens": 3252, + "total_tokens": 734886, + "reasoning_tokens": 2562 + }, + "turn_count": 13 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 84.4476706981659, + "token_usage": { + "input_tokens": 602674, + "output_tokens": 3535, + "total_tokens": 606209, + "reasoning_tokens": 2405 + }, + "turn_count": 16 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/reddit__llm_research_summary.json b/model_results/grok-4-fast/reddit__llm_research_summary.json new file mode 100644 index 00000000..827d8b84 --- /dev/null +++ b/model_results/grok-4-fast/reddit__llm_research_summary.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "reddit__llm_research_summary", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 72.01405262947083, + "token_usage": { + "input_tokens": 610637, + "output_tokens": 4634, + "total_tokens": 615271, + "reasoning_tokens": 3781 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 80.41897535324097, + "token_usage": { + "input_tokens": 820520, + "output_tokens": 3751, + "total_tokens": 824271, + "reasoning_tokens": 2774 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 68.50351619720459, + "token_usage": { + "input_tokens": 591644, + "output_tokens": 4397, + "total_tokens": 596041, + "reasoning_tokens": 3566 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 60.905073165893555, + "token_usage": { + "input_tokens": 336646, + "output_tokens": 4254, + "total_tokens": 340900, + "reasoning_tokens": 3560 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/reddit__movie_reviewer_analysis.json b/model_results/grok-4-fast/reddit__movie_reviewer_analysis.json new file mode 100644 index 00000000..652f7601 --- /dev/null +++ b/model_results/grok-4-fast/reddit__movie_reviewer_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "reddit__movie_reviewer_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 104.28021621704102, + "token_usage": { + "input_tokens": 787367, + "output_tokens": 8412, + "total_tokens": 795779, + "reasoning_tokens": 7504 + }, + "turn_count": 17 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 69.53013396263123, + "token_usage": { + "input_tokens": 576625, + "output_tokens": 5392, + "total_tokens": 582017, + "reasoning_tokens": 4554 + }, + "turn_count": 15 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 63.78611612319946, + "token_usage": { + "input_tokens": 624287, + "output_tokens": 6413, + "total_tokens": 630700, + "reasoning_tokens": 5578 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 79.39650273323059, + "token_usage": { + "input_tokens": 871617, + "output_tokens": 5629, + "total_tokens": 877246, + "reasoning_tokens": 4663 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/reddit__nba_statistics_analysis.json b/model_results/grok-4-fast/reddit__nba_statistics_analysis.json new file mode 100644 index 00000000..769f0f29 --- /dev/null +++ b/model_results/grok-4-fast/reddit__nba_statistics_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "reddit__nba_statistics_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 145.343403339386, + "token_usage": { + "input_tokens": 2894579, + "output_tokens": 8570, + "total_tokens": 2903149, + "reasoning_tokens": 6254 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 151.66982221603394, + "token_usage": { + "input_tokens": 1697840, + "output_tokens": 10765, + "total_tokens": 1708605, + "reasoning_tokens": 8907 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 111.23613333702087, + "token_usage": { + "input_tokens": 1073525, + "output_tokens": 6759, + "total_tokens": 1080284, + "reasoning_tokens": 5411 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 86.50196313858032, + "token_usage": { + "input_tokens": 662372, + "output_tokens": 5806, + "total_tokens": 668178, + "reasoning_tokens": 4935 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/reddit__routine_tracker_forum.json b/model_results/grok-4-fast/reddit__routine_tracker_forum.json new file mode 100644 index 00000000..c6678f94 --- /dev/null +++ b/model_results/grok-4-fast/reddit__routine_tracker_forum.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "reddit__routine_tracker_forum", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 59.010971546173096, + "token_usage": { + "input_tokens": 1110184, + "output_tokens": 2823, + "total_tokens": 1113007, + "reasoning_tokens": 1952 + }, + "turn_count": 14 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 76.2082691192627, + "token_usage": { + "input_tokens": 1152880, + "output_tokens": 2697, + "total_tokens": 1155577, + "reasoning_tokens": 1848 + }, + "turn_count": 16 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 74.61527252197266, + "token_usage": { + "input_tokens": 1122447, + "output_tokens": 2244, + "total_tokens": 1124691, + "reasoning_tokens": 1413 + }, + "turn_count": 15 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 233.45671105384827, + "token_usage": { + "input_tokens": 1418334, + "output_tokens": 2465, + "total_tokens": 1420799, + "reasoning_tokens": 1578 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/security__rls_business_access.json b/model_results/grok-4-fast/security__rls_business_access.json new file mode 100644 index 00000000..1f46fbdf --- /dev/null +++ b/model_results/grok-4-fast/security__rls_business_access.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "security__rls_business_access", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 54.152281522750854, + "token_usage": { + "input_tokens": 103296, + "output_tokens": 6756, + "total_tokens": 110052, + "reasoning_tokens": 3199 + }, + "turn_count": 13 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 89.85063052177429, + "token_usage": { + "input_tokens": 56036, + "output_tokens": 7134, + "total_tokens": 63170, + "reasoning_tokens": 5240 + }, + "turn_count": 9 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 79.84797501564026, + "token_usage": { + "input_tokens": 147064, + "output_tokens": 5916, + "total_tokens": 152980, + "reasoning_tokens": 4262 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 110.54613542556763, + "token_usage": { + "input_tokens": 471402, + "output_tokens": 6553, + "total_tokens": 477955, + "reasoning_tokens": 3542 + }, + "turn_count": 43 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/security__user_permission_audit.json b/model_results/grok-4-fast/security__user_permission_audit.json new file mode 100644 index 00000000..c3ed0536 --- /dev/null +++ b/model_results/grok-4-fast/security__user_permission_audit.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "security__user_permission_audit", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 40.413169384002686, + "token_usage": { + "input_tokens": 81237, + "output_tokens": 5574, + "total_tokens": 86811, + "reasoning_tokens": 4487 + }, + "turn_count": 10 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 48.254390001297, + "token_usage": { + "input_tokens": 61618, + "output_tokens": 4853, + "total_tokens": 66471, + "reasoning_tokens": 3737 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 46.04936122894287, + "token_usage": { + "input_tokens": 66601, + "output_tokens": 5599, + "total_tokens": 72200, + "reasoning_tokens": 4557 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 33.74569845199585, + "token_usage": { + "input_tokens": 79851, + "output_tokens": 4185, + "total_tokens": 84036, + "reasoning_tokens": 2991 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/self_assessment__faq_column_layout.json b/model_results/grok-4-fast/self_assessment__faq_column_layout.json new file mode 100644 index 00000000..f709c9da --- /dev/null +++ b/model_results/grok-4-fast/self_assessment__faq_column_layout.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "self_assessment__faq_column_layout", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 80.69040131568909, + "token_usage": { + "input_tokens": 465097, + "output_tokens": 5657, + "total_tokens": 470754, + "reasoning_tokens": 3970 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 81.31278705596924, + "token_usage": { + "input_tokens": 300954, + "output_tokens": 4467, + "total_tokens": 305421, + "reasoning_tokens": 3541 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 75.64905261993408, + "token_usage": { + "input_tokens": 437946, + "output_tokens": 5118, + "total_tokens": 443064, + "reasoning_tokens": 3982 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 80.1673743724823, + "token_usage": { + "input_tokens": 522323, + "output_tokens": 6350, + "total_tokens": 528673, + "reasoning_tokens": 4941 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/self_assessment__hyperfocus_analysis_report.json b/model_results/grok-4-fast/self_assessment__hyperfocus_analysis_report.json new file mode 100644 index 00000000..1e1e1747 --- /dev/null +++ b/model_results/grok-4-fast/self_assessment__hyperfocus_analysis_report.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "self_assessment__hyperfocus_analysis_report", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 490.66569781303406, + "token_usage": { + "input_tokens": 228757, + "output_tokens": 84023, + "total_tokens": 312780, + "reasoning_tokens": 5616 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 87.10163140296936, + "token_usage": { + "input_tokens": 161990, + "output_tokens": 9092, + "total_tokens": 171082, + "reasoning_tokens": 7771 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 74.65599489212036, + "token_usage": { + "input_tokens": 193103, + "output_tokens": 6113, + "total_tokens": 199216, + "reasoning_tokens": 5126 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 74.88243842124939, + "token_usage": { + "input_tokens": 220460, + "output_tokens": 7749, + "total_tokens": 228209, + "reasoning_tokens": 7115 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/self_assessment__numbered_list_emojis.json b/model_results/grok-4-fast/self_assessment__numbered_list_emojis.json new file mode 100644 index 00000000..9a8f8a4d --- /dev/null +++ b/model_results/grok-4-fast/self_assessment__numbered_list_emojis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "self_assessment__numbered_list_emojis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 121.29287600517273, + "token_usage": { + "input_tokens": 717522, + "output_tokens": 10009, + "total_tokens": 727531, + "reasoning_tokens": 7553 + }, + "turn_count": 24 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 508.04330611228943, + "token_usage": { + "input_tokens": 927654, + "output_tokens": 11825, + "total_tokens": 939479, + "reasoning_tokens": 9298 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 553.093966960907, + "token_usage": { + "input_tokens": 226597, + "output_tokens": 103687, + "total_tokens": 330284, + "reasoning_tokens": 8157 + }, + "turn_count": 9 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 93.50998592376709, + "token_usage": { + "input_tokens": 276583, + "output_tokens": 9546, + "total_tokens": 286129, + "reasoning_tokens": 7042 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping__advanced_product_analysis.json b/model_results/grok-4-fast/shopping__advanced_product_analysis.json new file mode 100644 index 00000000..865b9cf3 --- /dev/null +++ b/model_results/grok-4-fast/shopping__advanced_product_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping__advanced_product_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 71.9132034778595, + "token_usage": { + "input_tokens": 900874, + "output_tokens": 2312, + "total_tokens": 903186, + "reasoning_tokens": 1610 + }, + "turn_count": 16 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 76.39111614227295, + "token_usage": { + "input_tokens": 1326885, + "output_tokens": 3656, + "total_tokens": 1330541, + "reasoning_tokens": 2802 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 63.126275300979614, + "token_usage": { + "input_tokens": 820299, + "output_tokens": 2765, + "total_tokens": 823064, + "reasoning_tokens": 2029 + }, + "turn_count": 15 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 67.79973006248474, + "token_usage": { + "input_tokens": 819706, + "output_tokens": 2768, + "total_tokens": 822474, + "reasoning_tokens": 2087 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping__gaming_accessories_analysis.json b/model_results/grok-4-fast/shopping__gaming_accessories_analysis.json new file mode 100644 index 00000000..f71fded2 --- /dev/null +++ b/model_results/grok-4-fast/shopping__gaming_accessories_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping__gaming_accessories_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 113.50308537483215, + "token_usage": { + "input_tokens": 1378782, + "output_tokens": 3499, + "total_tokens": 1382281, + "reasoning_tokens": 2346 + }, + "turn_count": 22 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 169.1634442806244, + "token_usage": { + "input_tokens": 2541389, + "output_tokens": 5463, + "total_tokens": 2546852, + "reasoning_tokens": 4037 + }, + "turn_count": 31 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 104.62143349647522, + "token_usage": { + "input_tokens": 1663069, + "output_tokens": 4436, + "total_tokens": 1667505, + "reasoning_tokens": 2992 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 106.61391472816467, + "token_usage": { + "input_tokens": 1581509, + "output_tokens": 4218, + "total_tokens": 1585727, + "reasoning_tokens": 2985 + }, + "turn_count": 24 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping__health_routine_optimization.json b/model_results/grok-4-fast/shopping__health_routine_optimization.json new file mode 100644 index 00000000..1fc56f32 --- /dev/null +++ b/model_results/grok-4-fast/shopping__health_routine_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping__health_routine_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 88.40928888320923, + "token_usage": { + "input_tokens": 1071847, + "output_tokens": 3604, + "total_tokens": 1075451, + "reasoning_tokens": 2693 + }, + "turn_count": 18 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 103.08896255493164, + "token_usage": { + "input_tokens": 1276853, + "output_tokens": 4802, + "total_tokens": 1281655, + "reasoning_tokens": 3802 + }, + "turn_count": 19 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 86.69021010398865, + "token_usage": { + "input_tokens": 1152264, + "output_tokens": 3728, + "total_tokens": 1155992, + "reasoning_tokens": 2727 + }, + "turn_count": 17 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 88.08992671966553, + "token_usage": { + "input_tokens": 1056042, + "output_tokens": 3398, + "total_tokens": 1059440, + "reasoning_tokens": 2386 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping__holiday_baking_competition.json b/model_results/grok-4-fast/shopping__holiday_baking_competition.json new file mode 100644 index 00000000..227eb797 --- /dev/null +++ b/model_results/grok-4-fast/shopping__holiday_baking_competition.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping__holiday_baking_competition", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 232.6989462375641, + "token_usage": { + "input_tokens": 4272955, + "output_tokens": 7613, + "total_tokens": 4280568, + "reasoning_tokens": 5721 + }, + "turn_count": 36 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 210.50627279281616, + "token_usage": { + "input_tokens": 3804506, + "output_tokens": 10585, + "total_tokens": 3815091, + "reasoning_tokens": 8147 + }, + "turn_count": 34 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 200.19812726974487, + "token_usage": { + "input_tokens": 4302778, + "output_tokens": 10271, + "total_tokens": 4313049, + "reasoning_tokens": 8285 + }, + "turn_count": 35 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 155.43351340293884, + "token_usage": { + "input_tokens": 2477101, + "output_tokens": 7072, + "total_tokens": 2484173, + "reasoning_tokens": 5675 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping__multi_category_budget_analysis.json b/model_results/grok-4-fast/shopping__multi_category_budget_analysis.json new file mode 100644 index 00000000..e96c9895 --- /dev/null +++ b/model_results/grok-4-fast/shopping__multi_category_budget_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping__multi_category_budget_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 251.94581246376038, + "token_usage": { + "input_tokens": 6628911, + "output_tokens": 10458, + "total_tokens": 6639369, + "reasoning_tokens": 8158 + }, + "turn_count": 49 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 194.21247267723083, + "token_usage": { + "input_tokens": 3190375, + "output_tokens": 10333, + "total_tokens": 3200708, + "reasoning_tokens": 8734 + }, + "turn_count": 32 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 182.10172986984253, + "token_usage": { + "input_tokens": 4155619, + "output_tokens": 10557, + "total_tokens": 4166176, + "reasoning_tokens": 9018 + }, + "turn_count": 35 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 176.0252137184143, + "token_usage": { + "input_tokens": 2255318, + "output_tokens": 7567, + "total_tokens": 2262885, + "reasoning_tokens": 6433 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping__printer_keyboard_search.json b/model_results/grok-4-fast/shopping__printer_keyboard_search.json new file mode 100644 index 00000000..9809d00d --- /dev/null +++ b/model_results/grok-4-fast/shopping__printer_keyboard_search.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping__printer_keyboard_search", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 64.0178050994873, + "token_usage": { + "input_tokens": 505777, + "output_tokens": 2920, + "total_tokens": 508697, + "reasoning_tokens": 2465 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 65.19870638847351, + "token_usage": { + "input_tokens": 663763, + "output_tokens": 3066, + "total_tokens": 666829, + "reasoning_tokens": 2300 + }, + "turn_count": 13 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 98.66196751594543, + "token_usage": { + "input_tokens": 1325602, + "output_tokens": 4775, + "total_tokens": 1330377, + "reasoning_tokens": 3774 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 96.52966380119324, + "token_usage": { + "input_tokens": 1180344, + "output_tokens": 4154, + "total_tokens": 1184498, + "reasoning_tokens": 3410 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping__running_shoes_purchase.json b/model_results/grok-4-fast/shopping__running_shoes_purchase.json new file mode 100644 index 00000000..8479044f --- /dev/null +++ b/model_results/grok-4-fast/shopping__running_shoes_purchase.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping__running_shoes_purchase", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 42.87510895729065, + "token_usage": { + "input_tokens": 271708, + "output_tokens": 2943, + "total_tokens": 274651, + "reasoning_tokens": 2474 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 72.09514212608337, + "token_usage": { + "input_tokens": 490081, + "output_tokens": 4020, + "total_tokens": 494101, + "reasoning_tokens": 3048 + }, + "turn_count": 11 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 61.30683755874634, + "token_usage": { + "input_tokens": 291238, + "output_tokens": 4838, + "total_tokens": 296076, + "reasoning_tokens": 4048 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 32.9167058467865, + "token_usage": { + "input_tokens": 152994, + "output_tokens": 1831, + "total_tokens": 154825, + "reasoning_tokens": 1431 + }, + "turn_count": 6 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping_admin__customer_segmentation_setup.json b/model_results/grok-4-fast/shopping_admin__customer_segmentation_setup.json new file mode 100644 index 00000000..995bdd7b --- /dev/null +++ b/model_results/grok-4-fast/shopping_admin__customer_segmentation_setup.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping_admin__customer_segmentation_setup", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 61.54241704940796, + "token_usage": { + "input_tokens": 418047, + "output_tokens": 2151, + "total_tokens": 420198, + "reasoning_tokens": 1467 + }, + "turn_count": 16 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 63.27823305130005, + "token_usage": { + "input_tokens": 473755, + "output_tokens": 1839, + "total_tokens": 475594, + "reasoning_tokens": 1036 + }, + "turn_count": 17 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 56.19239258766174, + "token_usage": { + "input_tokens": 342616, + "output_tokens": 2366, + "total_tokens": 344982, + "reasoning_tokens": 1610 + }, + "turn_count": 12 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 52.29262828826904, + "token_usage": { + "input_tokens": 385141, + "output_tokens": 2117, + "total_tokens": 387258, + "reasoning_tokens": 1370 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping_admin__fitness_promotion_strategy.json b/model_results/grok-4-fast/shopping_admin__fitness_promotion_strategy.json new file mode 100644 index 00000000..134926d1 --- /dev/null +++ b/model_results/grok-4-fast/shopping_admin__fitness_promotion_strategy.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping_admin__fitness_promotion_strategy", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 122.45012879371643, + "token_usage": { + "input_tokens": 1594248, + "output_tokens": 5811, + "total_tokens": 1600059, + "reasoning_tokens": 4760 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 104.87030553817749, + "token_usage": { + "input_tokens": 2509994, + "output_tokens": 5751, + "total_tokens": 2515745, + "reasoning_tokens": 4916 + }, + "turn_count": 15 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 135.1025629043579, + "token_usage": { + "input_tokens": 4181174, + "output_tokens": 6681, + "total_tokens": 4187855, + "reasoning_tokens": 5525 + }, + "turn_count": 24 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 394.6514859199524, + "token_usage": { + "input_tokens": 10221640, + "output_tokens": 11088, + "total_tokens": 10232728, + "reasoning_tokens": 9332 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping_admin__marketing_customer_analysis.json b/model_results/grok-4-fast/shopping_admin__marketing_customer_analysis.json new file mode 100644 index 00000000..16d60625 --- /dev/null +++ b/model_results/grok-4-fast/shopping_admin__marketing_customer_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping_admin__marketing_customer_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 80.241295337677, + "token_usage": { + "input_tokens": 586941, + "output_tokens": 3965, + "total_tokens": 590906, + "reasoning_tokens": 3002 + }, + "turn_count": 15 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 81.10791397094727, + "token_usage": { + "input_tokens": 708465, + "output_tokens": 3014, + "total_tokens": 711479, + "reasoning_tokens": 1995 + }, + "turn_count": 18 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 87.85989713668823, + "token_usage": { + "input_tokens": 712334, + "output_tokens": 3605, + "total_tokens": 715939, + "reasoning_tokens": 2365 + }, + "turn_count": 18 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 91.39982771873474, + "token_usage": { + "input_tokens": 759178, + "output_tokens": 3697, + "total_tokens": 762875, + "reasoning_tokens": 2621 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping_admin__ny_expansion_analysis.json b/model_results/grok-4-fast/shopping_admin__ny_expansion_analysis.json new file mode 100644 index 00000000..51a2ba84 --- /dev/null +++ b/model_results/grok-4-fast/shopping_admin__ny_expansion_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping_admin__ny_expansion_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 63.6463520526886, + "token_usage": { + "input_tokens": 324282, + "output_tokens": 3863, + "total_tokens": 328145, + "reasoning_tokens": 3178 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 58.401482343673706, + "token_usage": { + "input_tokens": 366178, + "output_tokens": 3486, + "total_tokens": 369664, + "reasoning_tokens": 2761 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 92.82743620872498, + "token_usage": { + "input_tokens": 1208374, + "output_tokens": 4410, + "total_tokens": 1212784, + "reasoning_tokens": 3431 + }, + "turn_count": 25 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 75.25112700462341, + "token_usage": { + "input_tokens": 381780, + "output_tokens": 3687, + "total_tokens": 385467, + "reasoning_tokens": 2734 + }, + "turn_count": 18 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping_admin__products_sales_analysis.json b/model_results/grok-4-fast/shopping_admin__products_sales_analysis.json new file mode 100644 index 00000000..5805b496 --- /dev/null +++ b/model_results/grok-4-fast/shopping_admin__products_sales_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping_admin__products_sales_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 145.67451119422913, + "token_usage": { + "input_tokens": 4776767, + "output_tokens": 5499, + "total_tokens": 4782266, + "reasoning_tokens": 4488 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 128.21493673324585, + "token_usage": { + "input_tokens": 2196565, + "output_tokens": 5231, + "total_tokens": 2201796, + "reasoning_tokens": 4470 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 125.284499168396, + "token_usage": { + "input_tokens": 3493634, + "output_tokens": 5669, + "total_tokens": 3499303, + "reasoning_tokens": 4902 + }, + "turn_count": 13 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 83.83782696723938, + "token_usage": { + "input_tokens": 887609, + "output_tokens": 3681, + "total_tokens": 891290, + "reasoning_tokens": 3335 + }, + "turn_count": 7 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping_admin__sales_inventory_analysis.json b/model_results/grok-4-fast/shopping_admin__sales_inventory_analysis.json new file mode 100644 index 00000000..68d1b38b --- /dev/null +++ b/model_results/grok-4-fast/shopping_admin__sales_inventory_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping_admin__sales_inventory_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 204.37532448768616, + "token_usage": { + "input_tokens": 7085073, + "output_tokens": 9530, + "total_tokens": 7094603, + "reasoning_tokens": 8545 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 219.33916592597961, + "token_usage": { + "input_tokens": 7638200, + "output_tokens": 8739, + "total_tokens": 7646939, + "reasoning_tokens": 7610 + }, + "turn_count": 19 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 95.73089456558228, + "token_usage": { + "input_tokens": 2470849, + "output_tokens": 3356, + "total_tokens": 2474205, + "reasoning_tokens": 2697 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 138.0698573589325, + "token_usage": { + "input_tokens": 2305589, + "output_tokens": 5552, + "total_tokens": 2311141, + "reasoning_tokens": 4664 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/shopping_admin__search_filtering_operations.json b/model_results/grok-4-fast/shopping_admin__search_filtering_operations.json new file mode 100644 index 00000000..db41103e --- /dev/null +++ b/model_results/grok-4-fast/shopping_admin__search_filtering_operations.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "shopping_admin__search_filtering_operations", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 92.69794154167175, + "token_usage": { + "input_tokens": 617412, + "output_tokens": 4644, + "total_tokens": 622056, + "reasoning_tokens": 3331 + }, + "turn_count": 19 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 137.17717957496643, + "token_usage": { + "input_tokens": 1260826, + "output_tokens": 6168, + "total_tokens": 1266994, + "reasoning_tokens": 4211 + }, + "turn_count": 22 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 127.40874028205872, + "token_usage": { + "input_tokens": 872212, + "output_tokens": 5947, + "total_tokens": 878159, + "reasoning_tokens": 4279 + }, + "turn_count": 21 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 105.42771601676941, + "token_usage": { + "input_tokens": 587191, + "output_tokens": 5029, + "total_tokens": 592220, + "reasoning_tokens": 3644 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/sports__baseball_player_analysis.json b/model_results/grok-4-fast/sports__baseball_player_analysis.json new file mode 100644 index 00000000..52ea1c4e --- /dev/null +++ b/model_results/grok-4-fast/sports__baseball_player_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "sports__baseball_player_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 106.88859868049622, + "token_usage": { + "input_tokens": 807104, + "output_tokens": 8638, + "total_tokens": 815742, + "reasoning_tokens": 5179 + }, + "turn_count": 42 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 129.7336893081665, + "token_usage": { + "input_tokens": 1047254, + "output_tokens": 9371, + "total_tokens": 1056625, + "reasoning_tokens": 6535 + }, + "turn_count": 47 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 98.44750738143921, + "token_usage": { + "input_tokens": 788054, + "output_tokens": 8278, + "total_tokens": 796332, + "reasoning_tokens": 5188 + }, + "turn_count": 39 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 92.44631671905518, + "token_usage": { + "input_tokens": 644706, + "output_tokens": 8405, + "total_tokens": 653111, + "reasoning_tokens": 5794 + }, + "turn_count": 34 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/sports__participant_report_optimization.json b/model_results/grok-4-fast/sports__participant_report_optimization.json new file mode 100644 index 00000000..c037ad76 --- /dev/null +++ b/model_results/grok-4-fast/sports__participant_report_optimization.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "sports__participant_report_optimization", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 58.26852893829346, + "token_usage": { + "input_tokens": 101091, + "output_tokens": 6738, + "total_tokens": 107829, + "reasoning_tokens": 4290 + }, + "turn_count": 10 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 64.63362216949463, + "token_usage": { + "input_tokens": 141053, + "output_tokens": 5087, + "total_tokens": 146140, + "reasoning_tokens": 2796 + }, + "turn_count": 14 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 60.86538004875183, + "token_usage": { + "input_tokens": 161705, + "output_tokens": 7298, + "total_tokens": 169003, + "reasoning_tokens": 4612 + }, + "turn_count": 14 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 30.700495719909668, + "token_usage": { + "input_tokens": 142979, + "output_tokens": 3425, + "total_tokens": 146404, + "reasoning_tokens": 2250 + }, + "turn_count": 14 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/sports__team_roster_management.json b/model_results/grok-4-fast/sports__team_roster_management.json new file mode 100644 index 00000000..278d9fd5 --- /dev/null +++ b/model_results/grok-4-fast/sports__team_roster_management.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "sports__team_roster_management", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 57.16362690925598, + "token_usage": { + "input_tokens": 254769, + "output_tokens": 6013, + "total_tokens": 260782, + "reasoning_tokens": 4531 + }, + "turn_count": 19 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 125.15681409835815, + "token_usage": { + "input_tokens": 738544, + "output_tokens": 9605, + "total_tokens": 748149, + "reasoning_tokens": 6000 + }, + "turn_count": 40 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 76.2571792602539, + "token_usage": { + "input_tokens": 262372, + "output_tokens": 6881, + "total_tokens": 269253, + "reasoning_tokens": 5080 + }, + "turn_count": 20 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 63.27751708030701, + "token_usage": { + "input_tokens": 284138, + "output_tokens": 6667, + "total_tokens": 290805, + "reasoning_tokens": 5088 + }, + "turn_count": 22 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/standard_operating_procedure__deployment_process_sop.json b/model_results/grok-4-fast/standard_operating_procedure__deployment_process_sop.json new file mode 100644 index 00000000..febf65d7 --- /dev/null +++ b/model_results/grok-4-fast/standard_operating_procedure__deployment_process_sop.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "standard_operating_procedure__deployment_process_sop", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 85.01771593093872, + "token_usage": { + "input_tokens": 166000, + "output_tokens": 8425, + "total_tokens": 174425, + "reasoning_tokens": 5554 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 61.74423956871033, + "token_usage": { + "input_tokens": 231409, + "output_tokens": 7155, + "total_tokens": 238564, + "reasoning_tokens": 5885 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 1787.9060208797455, + "token_usage": { + "input_tokens": 818201, + "output_tokens": 118904, + "total_tokens": 937105, + "reasoning_tokens": 4867 + }, + "turn_count": 8 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 733.1941394805908, + "token_usage": { + "input_tokens": 484239, + "output_tokens": 98344, + "total_tokens": 582583, + "reasoning_tokens": 6242 + }, + "turn_count": 20 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/standard_operating_procedure__section_organization.json b/model_results/grok-4-fast/standard_operating_procedure__section_organization.json new file mode 100644 index 00000000..e66a9412 --- /dev/null +++ b/model_results/grok-4-fast/standard_operating_procedure__section_organization.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "standard_operating_procedure__section_organization", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 98.02617001533508, + "token_usage": { + "input_tokens": 250330, + "output_tokens": 8864, + "total_tokens": 259194, + "reasoning_tokens": 7775 + }, + "turn_count": 13 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 143.87897062301636, + "token_usage": { + "input_tokens": 393034, + "output_tokens": 13363, + "total_tokens": 406397, + "reasoning_tokens": 11872 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 546.5974888801575, + "token_usage": { + "input_tokens": 247467, + "output_tokens": 101375, + "total_tokens": 348842, + "reasoning_tokens": 5100 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 18.57643699645996, + "token_usage": { + "input_tokens": 23417, + "output_tokens": 1419, + "total_tokens": 24836, + "reasoning_tokens": 1269 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/student_database__duplicate_name.json b/model_results/grok-4-fast/student_database__duplicate_name.json new file mode 100644 index 00000000..40185ab2 --- /dev/null +++ b/model_results/grok-4-fast/student_database__duplicate_name.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "student_database__duplicate_name", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 32.75327706336975, + "token_usage": { + "input_tokens": 18430, + "output_tokens": 5714, + "total_tokens": 24144, + "reasoning_tokens": 5231 + }, + "turn_count": 4 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 30.554624319076538, + "token_usage": { + "input_tokens": 59513, + "output_tokens": 4974, + "total_tokens": 64487, + "reasoning_tokens": 4323 + }, + "turn_count": 7 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 51.164172887802124, + "token_usage": { + "input_tokens": 30699, + "output_tokens": 7250, + "total_tokens": 37949, + "reasoning_tokens": 6379 + }, + "turn_count": 5 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 64.76227450370789, + "token_usage": { + "input_tokens": 18207, + "output_tokens": 5498, + "total_tokens": 23705, + "reasoning_tokens": 5015 + }, + "turn_count": 4 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/student_database__english_talent.json b/model_results/grok-4-fast/student_database__english_talent.json new file mode 100644 index 00000000..8c1efdee --- /dev/null +++ b/model_results/grok-4-fast/student_database__english_talent.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "student_database__english_talent", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 62.378363847732544, + "token_usage": { + "input_tokens": 718742, + "output_tokens": 5337, + "total_tokens": 724079, + "reasoning_tokens": 3814 + }, + "turn_count": 25 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 86.13072562217712, + "token_usage": { + "input_tokens": 293028, + "output_tokens": 15962, + "total_tokens": 308990, + "reasoning_tokens": 1711 + }, + "turn_count": 9 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 123.32902336120605, + "token_usage": { + "input_tokens": 366463, + "output_tokens": 19229, + "total_tokens": 385692, + "reasoning_tokens": 5750 + }, + "turn_count": 12 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 105.57179641723633, + "token_usage": { + "input_tokens": 169865, + "output_tokens": 14808, + "total_tokens": 184673, + "reasoning_tokens": 1966 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/student_database__gradebased_score.json b/model_results/grok-4-fast/student_database__gradebased_score.json new file mode 100644 index 00000000..62fe4276 --- /dev/null +++ b/model_results/grok-4-fast/student_database__gradebased_score.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "student_database__gradebased_score", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 45.984002113342285, + "token_usage": { + "input_tokens": 93941, + "output_tokens": 8397, + "total_tokens": 102338, + "reasoning_tokens": 1876 + }, + "turn_count": 6 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 75.5304503440857, + "token_usage": { + "input_tokens": 97202, + "output_tokens": 11624, + "total_tokens": 108826, + "reasoning_tokens": 1870 + }, + "turn_count": 6 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 47.36026859283447, + "token_usage": { + "input_tokens": 111324, + "output_tokens": 8113, + "total_tokens": 119437, + "reasoning_tokens": 1955 + }, + "turn_count": 6 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 61.18229293823242, + "token_usage": { + "input_tokens": 55746, + "output_tokens": 9800, + "total_tokens": 65546, + "reasoning_tokens": 3706 + }, + "turn_count": 5 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/team_projects__priority_tasks_table.json b/model_results/grok-4-fast/team_projects__priority_tasks_table.json new file mode 100644 index 00000000..bdc39dfc --- /dev/null +++ b/model_results/grok-4-fast/team_projects__priority_tasks_table.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "team_projects__priority_tasks_table", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 82.55078721046448, + "token_usage": { + "input_tokens": 164390, + "output_tokens": 7504, + "total_tokens": 171894, + "reasoning_tokens": 6439 + }, + "turn_count": 11 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 132.8717610836029, + "token_usage": { + "input_tokens": 253259, + "output_tokens": 11315, + "total_tokens": 264574, + "reasoning_tokens": 10007 + }, + "turn_count": 14 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 77.95200490951538, + "token_usage": { + "input_tokens": 322625, + "output_tokens": 7860, + "total_tokens": 330485, + "reasoning_tokens": 6586 + }, + "turn_count": 18 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 77.79194188117981, + "token_usage": { + "input_tokens": 265780, + "output_tokens": 5894, + "total_tokens": 271674, + "reasoning_tokens": 4943 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/team_projects__swap_tasks.json b/model_results/grok-4-fast/team_projects__swap_tasks.json new file mode 100644 index 00000000..26202a3d --- /dev/null +++ b/model_results/grok-4-fast/team_projects__swap_tasks.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "team_projects__swap_tasks", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 97.05389833450317, + "token_usage": { + "input_tokens": 430767, + "output_tokens": 7660, + "total_tokens": 438427, + "reasoning_tokens": 5036 + }, + "turn_count": 16 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 92.11198449134827, + "token_usage": { + "input_tokens": 741180, + "output_tokens": 6184, + "total_tokens": 747364, + "reasoning_tokens": 4620 + }, + "turn_count": 15 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 233.5587673187256, + "token_usage": { + "input_tokens": 2995570, + "output_tokens": 13676, + "total_tokens": 3009246, + "reasoning_tokens": 7964 + }, + "turn_count": 23 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 95.1660213470459, + "token_usage": { + "input_tokens": 869336, + "output_tokens": 6109, + "total_tokens": 875445, + "reasoning_tokens": 4064 + }, + "turn_count": 10 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/threestudio__code_locating.json b/model_results/grok-4-fast/threestudio__code_locating.json new file mode 100644 index 00000000..d458aafc --- /dev/null +++ b/model_results/grok-4-fast/threestudio__code_locating.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "threestudio__code_locating", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 20.748981714248657, + "token_usage": { + "input_tokens": 76010, + "output_tokens": 1570, + "total_tokens": 77580, + "reasoning_tokens": 988 + }, + "turn_count": 11 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 16.69061779975891, + "token_usage": { + "input_tokens": 43656, + "output_tokens": 1706, + "total_tokens": 45362, + "reasoning_tokens": 1025 + }, + "turn_count": 12 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 18.263986349105835, + "token_usage": { + "input_tokens": 31888, + "output_tokens": 939, + "total_tokens": 32827, + "reasoning_tokens": 600 + }, + "turn_count": 7 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 32.649340867996216, + "token_usage": { + "input_tokens": 155450, + "output_tokens": 1346, + "total_tokens": 156796, + "reasoning_tokens": 866 + }, + "turn_count": 9 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/threestudio__output_analysis.json b/model_results/grok-4-fast/threestudio__output_analysis.json new file mode 100644 index 00000000..0fd39a7d --- /dev/null +++ b/model_results/grok-4-fast/threestudio__output_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "threestudio__output_analysis", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 15.648688554763794, + "token_usage": { + "input_tokens": 55295, + "output_tokens": 1606, + "total_tokens": 56901, + "reasoning_tokens": 961 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 25.291381359100342, + "token_usage": { + "input_tokens": 60896, + "output_tokens": 2537, + "total_tokens": 63433, + "reasoning_tokens": 1739 + }, + "turn_count": 10 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 48.82315731048584, + "token_usage": { + "input_tokens": 97929, + "output_tokens": 6668, + "total_tokens": 104597, + "reasoning_tokens": 5840 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 26.38133478164673, + "token_usage": { + "input_tokens": 84549, + "output_tokens": 2716, + "total_tokens": 87265, + "reasoning_tokens": 1882 + }, + "turn_count": 11 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/threestudio__requirements_completion.json b/model_results/grok-4-fast/threestudio__requirements_completion.json new file mode 100644 index 00000000..435a841b --- /dev/null +++ b/model_results/grok-4-fast/threestudio__requirements_completion.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "threestudio__requirements_completion", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 63.79930901527405, + "token_usage": { + "input_tokens": 742874, + "output_tokens": 5431, + "total_tokens": 748305, + "reasoning_tokens": 3958 + }, + "turn_count": 25 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 48.032352447509766, + "token_usage": { + "input_tokens": 352580, + "output_tokens": 5079, + "total_tokens": 357659, + "reasoning_tokens": 4177 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 75.89592266082764, + "token_usage": { + "input_tokens": 698367, + "output_tokens": 6472, + "total_tokens": 704839, + "reasoning_tokens": 5011 + }, + "turn_count": 25 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 69.59379458427429, + "token_usage": { + "input_tokens": 272796, + "output_tokens": 4494, + "total_tokens": 277290, + "reasoning_tokens": 3472 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/toronto_guide__change_color.json b/model_results/grok-4-fast/toronto_guide__change_color.json new file mode 100644 index 00000000..ff19ce16 --- /dev/null +++ b/model_results/grok-4-fast/toronto_guide__change_color.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "toronto_guide__change_color", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 82.14975619316101, + "token_usage": { + "input_tokens": 229823, + "output_tokens": 7295, + "total_tokens": 237118, + "reasoning_tokens": 5626 + }, + "turn_count": 15 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 52.450515270233154, + "token_usage": { + "input_tokens": 208061, + "output_tokens": 5057, + "total_tokens": 213118, + "reasoning_tokens": 3675 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 97.22592878341675, + "token_usage": { + "input_tokens": 432263, + "output_tokens": 8678, + "total_tokens": 440941, + "reasoning_tokens": 5761 + }, + "turn_count": 17 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 61.066659927368164, + "token_usage": { + "input_tokens": 222132, + "output_tokens": 5490, + "total_tokens": 227622, + "reasoning_tokens": 3802 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/toronto_guide__weekend_adventure_planner.json b/model_results/grok-4-fast/toronto_guide__weekend_adventure_planner.json new file mode 100644 index 00000000..245187c6 --- /dev/null +++ b/model_results/grok-4-fast/toronto_guide__weekend_adventure_planner.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "notion", + "task": "toronto_guide__weekend_adventure_planner", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 140.93655252456665, + "token_usage": { + "input_tokens": 456862, + "output_tokens": 13596, + "total_tokens": 470458, + "reasoning_tokens": 7983 + }, + "turn_count": 18 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 74.48811268806458, + "token_usage": { + "input_tokens": 270007, + "output_tokens": 7599, + "total_tokens": 277606, + "reasoning_tokens": 4867 + }, + "turn_count": 13 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 126.54843378067017, + "token_usage": { + "input_tokens": 392261, + "output_tokens": 13488, + "total_tokens": 405749, + "reasoning_tokens": 8195 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 97.88291597366333, + "token_usage": { + "input_tokens": 475537, + "output_tokens": 6658, + "total_tokens": 482195, + "reasoning_tokens": 4122 + }, + "turn_count": 26 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/vectors__dba_vector_analysis.json b/model_results/grok-4-fast/vectors__dba_vector_analysis.json new file mode 100644 index 00000000..a89d3939 --- /dev/null +++ b/model_results/grok-4-fast/vectors__dba_vector_analysis.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "postgres", + "task": "vectors__dba_vector_analysis", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 58.26984977722168, + "token_usage": { + "input_tokens": 232828, + "output_tokens": 5869, + "total_tokens": 238697, + "reasoning_tokens": 3992 + }, + "turn_count": 25 + }, + "run-2": { + "success": true, + "error_message": null, + "execution_time": 100.25369143486023, + "token_usage": { + "input_tokens": 256621, + "output_tokens": 7495, + "total_tokens": 264116, + "reasoning_tokens": 4913 + }, + "turn_count": 29 + }, + "run-3": { + "success": true, + "error_message": null, + "execution_time": 68.00967717170715, + "token_usage": { + "input_tokens": 228155, + "output_tokens": 6086, + "total_tokens": 234241, + "reasoning_tokens": 3317 + }, + "turn_count": 26 + }, + "run-4": { + "success": true, + "error_message": null, + "execution_time": 49.86398458480835, + "token_usage": { + "input_tokens": 132589, + "output_tokens": 5718, + "total_tokens": 138307, + "reasoning_tokens": 2610 + }, + "turn_count": 19 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/votenet__dataset_comparison.json b/model_results/grok-4-fast/votenet__dataset_comparison.json new file mode 100644 index 00000000..c11d9b51 --- /dev/null +++ b/model_results/grok-4-fast/votenet__dataset_comparison.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "votenet__dataset_comparison", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 117.02768898010254, + "token_usage": { + "input_tokens": 331066, + "output_tokens": 12728, + "total_tokens": 343794, + "reasoning_tokens": 11964 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 85.15126180648804, + "token_usage": { + "input_tokens": 400271, + "output_tokens": 8059, + "total_tokens": 408330, + "reasoning_tokens": 7180 + }, + "turn_count": 16 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 76.53135967254639, + "token_usage": { + "input_tokens": 343517, + "output_tokens": 7495, + "total_tokens": 351012, + "reasoning_tokens": 6883 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 102.4737012386322, + "token_usage": { + "input_tokens": 406958, + "output_tokens": 10420, + "total_tokens": 417378, + "reasoning_tokens": 9592 + }, + "turn_count": 15 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/votenet__debugging.json b/model_results/grok-4-fast/votenet__debugging.json new file mode 100644 index 00000000..737ee925 --- /dev/null +++ b/model_results/grok-4-fast/votenet__debugging.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "votenet__debugging", + "runs": { + "run-1": { + "success": true, + "error_message": null, + "execution_time": 21.337174654006958, + "token_usage": { + "input_tokens": 50190, + "output_tokens": 2323, + "total_tokens": 52513, + "reasoning_tokens": 1847 + }, + "turn_count": 9 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 33.43651866912842, + "token_usage": { + "input_tokens": 37346, + "output_tokens": 3773, + "total_tokens": 41119, + "reasoning_tokens": 3235 + }, + "turn_count": 8 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 34.03437638282776, + "token_usage": { + "input_tokens": 67759, + "output_tokens": 3750, + "total_tokens": 71509, + "reasoning_tokens": 3162 + }, + "turn_count": 10 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 42.30304169654846, + "token_usage": { + "input_tokens": 96247, + "output_tokens": 4189, + "total_tokens": 100436, + "reasoning_tokens": 3486 + }, + "turn_count": 13 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/votenet__requirements_writing.json b/model_results/grok-4-fast/votenet__requirements_writing.json new file mode 100644 index 00000000..b5a3ef35 --- /dev/null +++ b/model_results/grok-4-fast/votenet__requirements_writing.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "filesystem", + "task": "votenet__requirements_writing", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 51.89892864227295, + "token_usage": { + "input_tokens": 680433, + "output_tokens": 3097, + "total_tokens": 683530, + "reasoning_tokens": 1523 + }, + "turn_count": 28 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 71.68468761444092, + "token_usage": { + "input_tokens": 789156, + "output_tokens": 3974, + "total_tokens": 793130, + "reasoning_tokens": 2217 + }, + "turn_count": 32 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 63.722352266311646, + "token_usage": { + "input_tokens": 853698, + "output_tokens": 3486, + "total_tokens": 857184, + "reasoning_tokens": 1624 + }, + "turn_count": 33 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 48.874046087265015, + "token_usage": { + "input_tokens": 208979, + "output_tokens": 4076, + "total_tokens": 213055, + "reasoning_tokens": 2501 + }, + "turn_count": 8 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/web_search__birth_of_arvinxu.json b/model_results/grok-4-fast/web_search__birth_of_arvinxu.json new file mode 100644 index 00000000..7836f2a3 --- /dev/null +++ b/model_results/grok-4-fast/web_search__birth_of_arvinxu.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "web_search__birth_of_arvinxu", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 66.74845910072327, + "token_usage": { + "input_tokens": 338317, + "output_tokens": 3629, + "total_tokens": 341946, + "reasoning_tokens": 3092 + }, + "turn_count": 14 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 112.36713004112244, + "token_usage": { + "input_tokens": 956341, + "output_tokens": 4189, + "total_tokens": 960530, + "reasoning_tokens": 3461 + }, + "turn_count": 20 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 95.51250839233398, + "token_usage": { + "input_tokens": 432784, + "output_tokens": 4018, + "total_tokens": 436802, + "reasoning_tokens": 3365 + }, + "turn_count": 16 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 129.6201150417328, + "token_usage": { + "input_tokens": 823807, + "output_tokens": 4200, + "total_tokens": 828007, + "reasoning_tokens": 3553 + }, + "turn_count": 17 + } + } +} \ No newline at end of file diff --git a/model_results/grok-4-fast/web_search__r1_arxiv.json b/model_results/grok-4-fast/web_search__r1_arxiv.json new file mode 100644 index 00000000..90e0e4c6 --- /dev/null +++ b/model_results/grok-4-fast/web_search__r1_arxiv.json @@ -0,0 +1,55 @@ +{ + "model": "grok-4-fast", + "service": "playwright", + "task": "web_search__r1_arxiv", + "runs": { + "run-1": { + "success": false, + "error_message": null, + "execution_time": 53.77536916732788, + "token_usage": { + "input_tokens": 618356, + "output_tokens": 3224, + "total_tokens": 621580, + "reasoning_tokens": 2392 + }, + "turn_count": 8 + }, + "run-2": { + "success": false, + "error_message": null, + "execution_time": 92.56545186042786, + "token_usage": { + "input_tokens": 1497265, + "output_tokens": 4750, + "total_tokens": 1502015, + "reasoning_tokens": 3580 + }, + "turn_count": 17 + }, + "run-3": { + "success": false, + "error_message": null, + "execution_time": 69.89897394180298, + "token_usage": { + "input_tokens": 916326, + "output_tokens": 4025, + "total_tokens": 920351, + "reasoning_tokens": 3114 + }, + "turn_count": 11 + }, + "run-4": { + "success": false, + "error_message": null, + "execution_time": 76.14770317077637, + "token_usage": { + "input_tokens": 695510, + "output_tokens": 3684, + "total_tokens": 699194, + "reasoning_tokens": 2672 + }, + "turn_count": 12 + } + } +} \ No newline at end of file diff --git a/summary.json b/summary.json index 5b1e587d..aa77530c 100644 --- a/summary.json +++ b/summary.json @@ -1,7 +1,32 @@ { - "generated_at": "2025-09-18T12:47:26.868861", + "generated_at": "2025-11-26T15:49:34.853971", "k": 4, "overall": { + "deepseek-v3-1-terminus-thinking": { + "total_tasks": 127, + "total_agent_execution_time": 373135.9043517113, + "total_input_tokens": 172323078, + "total_output_tokens": 7437158, + "total_tokens": 179760236, + "total_turns": 6603, + "avg_agent_execution_time": 734.5195, + "avg_input_tokens": 339218.6575, + "avg_output_tokens": 14640.0748, + "avg_total_tokens": 353858.7323, + "avg_turns": 12.998, + "per_run_input_tokens": 43080769.5, + "per_run_output_tokens": 1859289.5, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus-thinking", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.2126, + "std": 0.0329 + }, + "pass@4": 0.3701, + "pass^4": 0.0551 + }, "deepseek-chat": { "total_tasks": 127, "total_agent_execution_time": 137112.82717490196, @@ -52,29 +77,6 @@ "pass@4": 0.126, "pass^4": 0.0315 }, - "claude-opus-4-1": { - "total_tasks": 127, - "total_agent_execution_time": 45950.07896590233, - "total_input_tokens": 74430756, - "total_output_tokens": 653210, - "total_tokens": 75083966, - "total_turns": 2214, - "avg_agent_execution_time": 361.8116, - "avg_input_tokens": 586068.9449, - "avg_output_tokens": 5143.3858, - "avg_total_tokens": 591212.3307, - "avg_turns": 17.4331, - "per_run_input_tokens": 74430756.0, - "per_run_output_tokens": 653210.0, - "per_run_cost": 1165.45209, - "actual_model_name": "claude-opus-4-1-20250805", - "is_open_source_model": false, - "is_reasoning_model": false, - "pass@1": { - "avg": 0.2992, - "std": 0.0 - } - }, "glm-4-5": { "total_tasks": 127, "total_agent_execution_time": 84480.56672096252, @@ -125,6 +127,31 @@ "pass@4": 0.4331, "pass^4": 0.126 }, + "claude-sonnet-4-5": { + "total_tasks": 127, + "total_agent_execution_time": 87988.92542219162, + "total_input_tokens": 359671392, + "total_output_tokens": 3158899, + "total_tokens": 362830291, + "total_turns": 8516, + "avg_agent_execution_time": 173.2065, + "avg_input_tokens": 708014.5512, + "avg_output_tokens": 6218.3051, + "avg_total_tokens": 714232.8563, + "avg_turns": 16.7638, + "per_run_input_tokens": 89917848.0, + "per_run_output_tokens": 789724.75, + "per_run_cost": null, + "actual_model_name": "claude-sonnet-4-5-20250929", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.3209, + "std": 0.0232 + }, + "pass@4": 0.4646, + "pass^4": 0.1654 + }, "claude-sonnet-4-low": { "total_tasks": 127, "total_agent_execution_time": 101288.96435308456, @@ -250,6 +277,31 @@ "pass@4": 0.1024, "pass^4": 0.0079 }, + "glm-4-6": { + "total_tasks": 127, + "total_agent_execution_time": 123439.66112017632, + "total_input_tokens": 390392448, + "total_output_tokens": 2338366, + "total_tokens": 392730814, + "total_turns": 11403, + "avg_agent_execution_time": 242.9915, + "avg_input_tokens": 768489.0709, + "avg_output_tokens": 4603.0827, + "avg_total_tokens": 773092.1535, + "avg_turns": 22.4469, + "per_run_input_tokens": 97598112.0, + "per_run_output_tokens": 584591.5, + "per_run_cost": null, + "actual_model_name": "", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.2362, + "std": 0.0147 + }, + "pass@4": 0.3622, + "pass^4": 0.1339 + }, "gpt-5-medium": { "total_tasks": 127, "total_agent_execution_time": 242931.40538144112, @@ -475,6 +527,31 @@ "pass@4": 0.189, "pass^4": 0.0079 }, + "deepseek-v3-1-terminus": { + "total_tasks": 127, + "total_agent_execution_time": 124399.72146391869, + "total_input_tokens": 236208725, + "total_output_tokens": 1266817, + "total_tokens": 237475542, + "total_turns": 7753, + "avg_agent_execution_time": 244.8813, + "avg_input_tokens": 464977.8051, + "avg_output_tokens": 2493.7343, + "avg_total_tokens": 467471.5394, + "avg_turns": 15.2618, + "per_run_input_tokens": 59052181.25, + "per_run_output_tokens": 316704.25, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.1654, + "std": 0.0507 + }, + "pass@4": 0.2992, + "pass^4": 0.0394 + }, "qwen-3-max": { "total_tasks": 127, "total_agent_execution_time": 108499.87574696541, @@ -675,6 +752,31 @@ "pass@4": 0.1339, "pass^4": 0.0 }, + "grok-4-fast": { + "total_tasks": 127, + "total_agent_execution_time": 55840.35218334198, + "total_input_tokens": 346217495, + "total_output_tokens": 4517025, + "total_tokens": 350734520, + "total_turns": 7873, + "avg_agent_execution_time": 109.922, + "avg_input_tokens": 681530.502, + "avg_output_tokens": 8891.7815, + "avg_total_tokens": 690422.2835, + "avg_turns": 15.498, + "per_run_input_tokens": 86554373.75, + "per_run_output_tokens": 1129256.25, + "per_run_cost": null, + "actual_model_name": "grok-4-fast-reasoning", + "is_open_source_model": false, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.2402, + "std": 0.0307 + }, + "pass@4": 0.3858, + "pass^4": 0.126 + }, "o4-mini": { "total_tasks": 127, "total_agent_execution_time": 164245.99844169617, @@ -727,6 +829,31 @@ } }, "filesystem": { + "deepseek-v3-1-terminus-thinking": { + "total_tasks": 30, + "total_agent_execution_time": 92813.15895938873, + "total_input_tokens": 12850849, + "total_output_tokens": 1807599, + "total_tokens": 14658448, + "total_turns": 1181, + "avg_agent_execution_time": 773.443, + "avg_input_tokens": 107090.4083, + "avg_output_tokens": 15063.325, + "avg_total_tokens": 122153.7333, + "avg_turns": 9.8417, + "per_run_input_tokens": 3212712.25, + "per_run_output_tokens": 451899.75, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus-thinking", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.2417, + "std": 0.101 + }, + "pass@4": 0.4333, + "pass^4": 0.0667 + }, "deepseek-chat": { "total_tasks": 30, "total_agent_execution_time": 33809.433807849884, @@ -777,29 +904,6 @@ "pass@4": 0.2, "pass^4": 0.0333 }, - "claude-opus-4-1": { - "total_tasks": 30, - "total_agent_execution_time": 8033.295060634613, - "total_input_tokens": 8164979, - "total_output_tokens": 130994, - "total_tokens": 8295973, - "total_turns": 491, - "avg_agent_execution_time": 267.7765, - "avg_input_tokens": 272165.9667, - "avg_output_tokens": 4366.4667, - "avg_total_tokens": 276532.4333, - "avg_turns": 16.3667, - "per_run_input_tokens": 8164979.0, - "per_run_output_tokens": 130994.0, - "per_run_cost": 132.299235, - "actual_model_name": "claude-opus-4-1-20250805", - "is_open_source_model": false, - "is_reasoning_model": false, - "pass@1": { - "avg": 0.3333, - "std": 0.0 - } - }, "glm-4-5": { "total_tasks": 30, "total_agent_execution_time": 15624.920702934265, @@ -850,6 +954,31 @@ "pass@4": 0.5, "pass^4": 0.2667 }, + "claude-sonnet-4-5": { + "total_tasks": 30, + "total_agent_execution_time": 11409.54042005539, + "total_input_tokens": 32245290, + "total_output_tokens": 569223, + "total_tokens": 32814513, + "total_turns": 1328, + "avg_agent_execution_time": 95.0795, + "avg_input_tokens": 268710.75, + "avg_output_tokens": 4743.525, + "avg_total_tokens": 273454.275, + "avg_turns": 11.0667, + "per_run_input_tokens": 8061322.5, + "per_run_output_tokens": 142305.75, + "per_run_cost": null, + "actual_model_name": "claude-sonnet-4-5-20250929", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.325, + "std": 0.0493 + }, + "pass@4": 0.4333, + "pass^4": 0.1333 + }, "claude-sonnet-4-low": { "total_tasks": 30, "total_agent_execution_time": 21124.522393226624, @@ -975,6 +1104,31 @@ "pass@4": 0.3, "pass^4": 0.0333 }, + "glm-4-6": { + "total_tasks": 30, + "total_agent_execution_time": 21097.016854286194, + "total_input_tokens": 81414682, + "total_output_tokens": 626800, + "total_tokens": 82041482, + "total_turns": 2768, + "avg_agent_execution_time": 175.8085, + "avg_input_tokens": 678455.6833, + "avg_output_tokens": 5223.3333, + "avg_total_tokens": 683679.0167, + "avg_turns": 23.0667, + "per_run_input_tokens": 20353670.5, + "per_run_output_tokens": 156700.0, + "per_run_cost": null, + "actual_model_name": "", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.1, + "std": 0.0333 + }, + "pass@4": 0.1667, + "pass^4": 0.0333 + }, "gpt-5-medium": { "total_tasks": 30, "total_agent_execution_time": 37567.695655584335, @@ -1200,6 +1354,31 @@ "pass@4": 0.3333, "pass^4": 0.0333 }, + "deepseek-v3-1-terminus": { + "total_tasks": 30, + "total_agent_execution_time": 21577.44312930107, + "total_input_tokens": 23277761, + "total_output_tokens": 288361, + "total_tokens": 23566122, + "total_turns": 1554, + "avg_agent_execution_time": 179.812, + "avg_input_tokens": 193981.3417, + "avg_output_tokens": 2403.0083, + "avg_total_tokens": 196384.35, + "avg_turns": 12.95, + "per_run_input_tokens": 5819440.25, + "per_run_output_tokens": 72090.25, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.1083, + "std": 0.0493 + }, + "pass@4": 0.2, + "pass^4": 0.0333 + }, "qwen-3-max": { "total_tasks": 30, "total_agent_execution_time": 16070.112683296204, @@ -1400,6 +1579,31 @@ "pass@4": 0.1667, "pass^4": 0.0 }, + "grok-4-fast": { + "total_tasks": 30, + "total_agent_execution_time": 8968.955112218857, + "total_input_tokens": 63006836, + "total_output_tokens": 788211, + "total_tokens": 63795047, + "total_turns": 1539, + "avg_agent_execution_time": 74.7413, + "avg_input_tokens": 525056.9667, + "avg_output_tokens": 6568.425, + "avg_total_tokens": 531625.3917, + "avg_turns": 12.825, + "per_run_input_tokens": 15751709.0, + "per_run_output_tokens": 197052.75, + "per_run_cost": null, + "actual_model_name": "grok-4-fast-reasoning", + "is_open_source_model": false, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.2917, + "std": 0.0722 + }, + "pass@4": 0.5333, + "pass^4": 0.1667 + }, "o4-mini": { "total_tasks": 30, "total_agent_execution_time": 31632.499047517776, @@ -1452,6 +1656,31 @@ } }, "github": { + "deepseek-v3-1-terminus-thinking": { + "total_tasks": 23, + "total_agent_execution_time": 64614.89937853813, + "total_input_tokens": 34232908, + "total_output_tokens": 1397648, + "total_tokens": 35630556, + "total_turns": 798, + "avg_agent_execution_time": 702.3359, + "avg_input_tokens": 372096.8261, + "avg_output_tokens": 15191.8261, + "avg_total_tokens": 387288.6522, + "avg_turns": 8.6739, + "per_run_input_tokens": 8558227.0, + "per_run_output_tokens": 349412.0, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus-thinking", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.1087, + "std": 0.0486 + }, + "pass@4": 0.2174, + "pass^4": 0.0 + }, "deepseek-chat": { "total_tasks": 23, "total_agent_execution_time": 17843.541527748108, @@ -1502,29 +1731,6 @@ "pass@4": 0.087, "pass^4": 0.0435 }, - "claude-opus-4-1": { - "total_tasks": 23, - "total_agent_execution_time": 8974.225908994675, - "total_input_tokens": 14274491, - "total_output_tokens": 134233, - "total_tokens": 14408724, - "total_turns": 248, - "avg_agent_execution_time": 390.1837, - "avg_input_tokens": 620630.0435, - "avg_output_tokens": 5836.2174, - "avg_total_tokens": 626466.2609, - "avg_turns": 10.7826, - "per_run_input_tokens": 14274491.0, - "per_run_output_tokens": 134233.0, - "per_run_cost": 224.18484, - "actual_model_name": "claude-opus-4-1-20250805", - "is_open_source_model": false, - "is_reasoning_model": false, - "pass@1": { - "avg": 0.2174, - "std": 0.0 - } - }, "glm-4-5": { "total_tasks": 23, "total_agent_execution_time": 14126.248623132706, @@ -1575,6 +1781,31 @@ "pass@4": 0.2174, "pass^4": 0.0435 }, + "claude-sonnet-4-5": { + "total_tasks": 23, + "total_agent_execution_time": 20242.79474067688, + "total_input_tokens": 87531964, + "total_output_tokens": 827074, + "total_tokens": 88359038, + "total_turns": 1495, + "avg_agent_execution_time": 220.0304, + "avg_input_tokens": 951434.3913, + "avg_output_tokens": 8989.9348, + "avg_total_tokens": 960424.3261, + "avg_turns": 16.25, + "per_run_input_tokens": 21882991.0, + "per_run_output_tokens": 206768.5, + "per_run_cost": null, + "actual_model_name": "claude-sonnet-4-5-20250929", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.2935, + "std": 0.089 + }, + "pass@4": 0.4348, + "pass^4": 0.1739 + }, "claude-sonnet-4-low": { "total_tasks": 23, "total_agent_execution_time": 15970.095800638199, @@ -1700,6 +1931,31 @@ "pass@4": 0.0, "pass^4": 0.0 }, + "glm-4-6": { + "total_tasks": 23, + "total_agent_execution_time": 19691.034957170486, + "total_input_tokens": 92360286, + "total_output_tokens": 372028, + "total_tokens": 92732314, + "total_turns": 2030, + "avg_agent_execution_time": 214.033, + "avg_input_tokens": 1003916.1522, + "avg_output_tokens": 4043.7826, + "avg_total_tokens": 1007959.9348, + "avg_turns": 22.0652, + "per_run_input_tokens": 23090071.5, + "per_run_output_tokens": 93007.0, + "per_run_cost": null, + "actual_model_name": "", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.1957, + "std": 0.0217 + }, + "pass@4": 0.3043, + "pass^4": 0.1304 + }, "gpt-5-medium": { "total_tasks": 23, "total_agent_execution_time": 42011.101893901825, @@ -1925,6 +2181,31 @@ "pass@4": 0.1304, "pass^4": 0.0 }, + "deepseek-v3-1-terminus": { + "total_tasks": 23, + "total_agent_execution_time": 21333.615911722183, + "total_input_tokens": 41122717, + "total_output_tokens": 183454, + "total_tokens": 41306171, + "total_turns": 835, + "avg_agent_execution_time": 231.8871, + "avg_input_tokens": 446986.0543, + "avg_output_tokens": 1994.0652, + "avg_total_tokens": 448980.1196, + "avg_turns": 9.0761, + "per_run_input_tokens": 10280679.25, + "per_run_output_tokens": 45863.5, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.0543, + "std": 0.0713 + }, + "pass@4": 0.1739, + "pass^4": 0.0 + }, "qwen-3-max": { "total_tasks": 23, "total_agent_execution_time": 16698.327759742737, @@ -2125,6 +2406,31 @@ "pass@4": 0.087, "pass^4": 0.0 }, + "grok-4-fast": { + "total_tasks": 23, + "total_agent_execution_time": 13161.74501156807, + "total_input_tokens": 52963591, + "total_output_tokens": 992874, + "total_tokens": 53956465, + "total_turns": 1047, + "avg_agent_execution_time": 143.0624, + "avg_input_tokens": 575691.2065, + "avg_output_tokens": 10792.1087, + "avg_total_tokens": 586483.3152, + "avg_turns": 11.3804, + "per_run_input_tokens": 13240897.75, + "per_run_output_tokens": 248218.5, + "per_run_cost": null, + "actual_model_name": "grok-4-fast-reasoning", + "is_open_source_model": false, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.1304, + "std": 0.0307 + }, + "pass@4": 0.2174, + "pass^4": 0.0 + }, "o4-mini": { "total_tasks": 23, "total_agent_execution_time": 22887.657103300095, @@ -2177,6 +2483,31 @@ } }, "notion": { + "deepseek-v3-1-terminus-thinking": { + "total_tasks": 28, + "total_agent_execution_time": 102976.53085327148, + "total_input_tokens": 49897002, + "total_output_tokens": 2182951, + "total_tokens": 52079953, + "total_turns": 1664, + "avg_agent_execution_time": 919.4333, + "avg_input_tokens": 445508.9464, + "avg_output_tokens": 19490.6339, + "avg_total_tokens": 464999.5804, + "avg_turns": 14.8571, + "per_run_input_tokens": 12474250.5, + "per_run_output_tokens": 545737.75, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus-thinking", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.2232, + "std": 0.0296 + }, + "pass@4": 0.3929, + "pass^4": 0.0357 + }, "deepseek-chat": { "total_tasks": 28, "total_agent_execution_time": 26761.268040657043, @@ -2227,29 +2558,6 @@ "pass@4": 0.1429, "pass^4": 0.0 }, - "claude-opus-4-1": { - "total_tasks": 28, - "total_agent_execution_time": 8238.807499885559, - "total_input_tokens": 17865599, - "total_output_tokens": 110144, - "total_tokens": 17975743, - "total_turns": 477, - "avg_agent_execution_time": 294.2431, - "avg_input_tokens": 638057.1071, - "avg_output_tokens": 3933.7143, - "avg_total_tokens": 641990.8214, - "avg_turns": 17.0357, - "per_run_input_tokens": 17865599.0, - "per_run_output_tokens": 110144.0, - "per_run_cost": 276.244785, - "actual_model_name": "claude-opus-4-1-20250805", - "is_open_source_model": false, - "is_reasoning_model": false, - "pass@1": { - "avg": 0.3571, - "std": 0.0 - } - }, "glm-4-5": { "total_tasks": 28, "total_agent_execution_time": 24882.952006816864, @@ -2300,6 +2608,31 @@ "pass@4": 0.4643, "pass^4": 0.0714 }, + "claude-sonnet-4-5": { + "total_tasks": 28, + "total_agent_execution_time": 18484.7717359066, + "total_input_tokens": 75120675, + "total_output_tokens": 545767, + "total_tokens": 75666442, + "total_turns": 1615, + "avg_agent_execution_time": 165.0426, + "avg_input_tokens": 670720.3125, + "avg_output_tokens": 4872.9196, + "avg_total_tokens": 675593.2321, + "avg_turns": 14.4196, + "per_run_input_tokens": 18780168.75, + "per_run_output_tokens": 136441.75, + "per_run_cost": null, + "actual_model_name": "claude-sonnet-4-5-20250929", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.25, + "std": 0.0758 + }, + "pass@4": 0.4643, + "pass^4": 0.0357 + }, "claude-sonnet-4-low": { "total_tasks": 28, "total_agent_execution_time": 24645.00325536728, @@ -2425,6 +2758,31 @@ "pass@4": 0.0, "pass^4": 0.0 }, + "glm-4-6": { + "total_tasks": 28, + "total_agent_execution_time": 33522.80669927597, + "total_input_tokens": 100047124, + "total_output_tokens": 534511, + "total_tokens": 100581635, + "total_turns": 2824, + "avg_agent_execution_time": 299.3108, + "avg_input_tokens": 893277.8929, + "avg_output_tokens": 4772.4196, + "avg_total_tokens": 898050.3125, + "avg_turns": 25.2143, + "per_run_input_tokens": 25011781.0, + "per_run_output_tokens": 133627.75, + "per_run_cost": null, + "actual_model_name": "", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.25, + "std": 0.0437 + }, + "pass@4": 0.3929, + "pass^4": 0.1429 + }, "gpt-5-medium": { "total_tasks": 28, "total_agent_execution_time": 74123.09180688858, @@ -2650,6 +3008,31 @@ "pass@4": 0.1429, "pass^4": 0.0 }, + "deepseek-v3-1-terminus": { + "total_tasks": 28, + "total_agent_execution_time": 28280.00402736664, + "total_input_tokens": 80265896, + "total_output_tokens": 312143, + "total_tokens": 80578039, + "total_turns": 2153, + "avg_agent_execution_time": 252.5, + "avg_input_tokens": 716659.7857, + "avg_output_tokens": 2786.9911, + "avg_total_tokens": 719446.7768, + "avg_turns": 19.2232, + "per_run_input_tokens": 20066474.0, + "per_run_output_tokens": 78035.75, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.2232, + "std": 0.0813 + }, + "pass@4": 0.3929, + "pass^4": 0.0714 + }, "qwen-3-max": { "total_tasks": 28, "total_agent_execution_time": 20556.586127758026, @@ -2850,6 +3233,31 @@ "pass@4": 0.1429, "pass^4": 0.0 }, + "grok-4-fast": { + "total_tasks": 28, + "total_agent_execution_time": 17106.341089487076, + "total_input_tokens": 59274108, + "total_output_tokens": 1683861, + "total_tokens": 60957969, + "total_turns": 1936, + "avg_agent_execution_time": 152.7352, + "avg_input_tokens": 529233.1071, + "avg_output_tokens": 15034.4732, + "avg_total_tokens": 544267.5804, + "avg_turns": 17.2857, + "per_run_input_tokens": 14818527.0, + "per_run_output_tokens": 420965.25, + "per_run_cost": null, + "actual_model_name": "grok-4-fast-reasoning", + "is_open_source_model": false, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.0357, + "std": 0.0 + }, + "pass@4": 0.0357, + "pass^4": 0.0357 + }, "o4-mini": { "total_tasks": 28, "total_agent_execution_time": 49544.87328624725, @@ -2902,6 +3310,31 @@ } }, "playwright": { + "deepseek-v3-1-terminus-thinking": { + "total_tasks": 25, + "total_agent_execution_time": 77575.21338820457, + "total_input_tokens": 53019890, + "total_output_tokens": 1369290, + "total_tokens": 54389180, + "total_turns": 1393, + "avg_agent_execution_time": 775.7521, + "avg_input_tokens": 530198.9, + "avg_output_tokens": 13692.9, + "avg_total_tokens": 543891.8, + "avg_turns": 13.93, + "per_run_input_tokens": 13254972.5, + "per_run_output_tokens": 342322.5, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus-thinking", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.09, + "std": 0.0173 + }, + "pass@4": 0.2, + "pass^4": 0.0 + }, "deepseek-chat": { "total_tasks": 25, "total_agent_execution_time": 28832.343264341354, @@ -2952,29 +3385,6 @@ "pass@4": 0.12, "pass^4": 0.04 }, - "claude-opus-4-1": { - "total_tasks": 25, - "total_agent_execution_time": 9880.799889326096, - "total_input_tokens": 28651348, - "total_output_tokens": 72107, - "total_tokens": 28723455, - "total_turns": 476, - "avg_agent_execution_time": 395.232, - "avg_input_tokens": 1146053.92, - "avg_output_tokens": 2884.28, - "avg_total_tokens": 1148938.2, - "avg_turns": 19.04, - "per_run_input_tokens": 28651348.0, - "per_run_output_tokens": 72107.0, - "per_run_cost": 435.178245, - "actual_model_name": "claude-opus-4-1-20250805", - "is_open_source_model": false, - "is_reasoning_model": false, - "pass@1": { - "avg": 0.24, - "std": 0.0 - } - }, "glm-4-5": { "total_tasks": 25, "total_agent_execution_time": 16563.084107398987, @@ -3025,6 +3435,31 @@ "pass@4": 0.32, "pass^4": 0.08 }, + "claude-sonnet-4-5": { + "total_tasks": 25, + "total_agent_execution_time": 17537.452577114105, + "total_input_tokens": 124180757, + "total_output_tokens": 329104, + "total_tokens": 124509861, + "total_turns": 1943, + "avg_agent_execution_time": 175.3745, + "avg_input_tokens": 1241807.57, + "avg_output_tokens": 3291.04, + "avg_total_tokens": 1245098.61, + "avg_turns": 19.43, + "per_run_input_tokens": 31045189.25, + "per_run_output_tokens": 82276.0, + "per_run_cost": null, + "actual_model_name": "claude-sonnet-4-5-20250929", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.27, + "std": 0.0592 + }, + "pass@4": 0.36, + "pass^4": 0.16 + }, "claude-sonnet-4-low": { "total_tasks": 25, "total_agent_execution_time": 23905.656344890594, @@ -3150,6 +3585,31 @@ "pass@4": 0.0, "pass^4": 0.0 }, + "glm-4-6": { + "total_tasks": 25, + "total_agent_execution_time": 27030.332295656204, + "total_input_tokens": 86267883, + "total_output_tokens": 361118, + "total_tokens": 86629001, + "total_turns": 1696, + "avg_agent_execution_time": 270.3033, + "avg_input_tokens": 862678.83, + "avg_output_tokens": 3611.18, + "avg_total_tokens": 866290.01, + "avg_turns": 16.96, + "per_run_input_tokens": 21566970.75, + "per_run_output_tokens": 90279.5, + "per_run_cost": null, + "actual_model_name": "", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.19, + "std": 0.052 + }, + "pass@4": 0.32, + "pass^4": 0.12 + }, "gpt-5-medium": { "total_tasks": 25, "total_agent_execution_time": 60819.73624634743, @@ -3375,6 +3835,31 @@ "pass@4": 0.04, "pass^4": 0.0 }, + "deepseek-v3-1-terminus": { + "total_tasks": 25, + "total_agent_execution_time": 32988.77002954483, + "total_input_tokens": 67216694, + "total_output_tokens": 187775, + "total_tokens": 67404469, + "total_turns": 1559, + "avg_agent_execution_time": 329.8877, + "avg_input_tokens": 672166.94, + "avg_output_tokens": 1877.75, + "avg_total_tokens": 674044.69, + "avg_turns": 15.59, + "per_run_input_tokens": 16804173.5, + "per_run_output_tokens": 46943.75, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.13, + "std": 0.0173 + }, + "pass@4": 0.2, + "pass^4": 0.08 + }, "qwen-3-max": { "total_tasks": 25, "total_agent_execution_time": 41767.95633363724, @@ -3575,6 +4060,31 @@ "pass@4": 0.04, "pass^4": 0.0 }, + "grok-4-fast": { + "total_tasks": 25, + "total_agent_execution_time": 10575.462986469269, + "total_input_tokens": 152417879, + "total_output_tokens": 492539, + "total_tokens": 152910418, + "total_turns": 1792, + "avg_agent_execution_time": 105.7546, + "avg_input_tokens": 1524178.79, + "avg_output_tokens": 4925.39, + "avg_total_tokens": 1529104.18, + "avg_turns": 17.92, + "per_run_input_tokens": 38104469.75, + "per_run_output_tokens": 123134.75, + "per_run_cost": null, + "actual_model_name": "grok-4-fast-reasoning", + "is_open_source_model": false, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.27, + "std": 0.0332 + }, + "pass@4": 0.4, + "pass^4": 0.16 + }, "o4-mini": { "total_tasks": 25, "total_agent_execution_time": 53063.511632204056, @@ -3627,6 +4137,31 @@ } }, "postgres": { + "deepseek-v3-1-terminus-thinking": { + "total_tasks": 21, + "total_agent_execution_time": 35156.10177230835, + "total_input_tokens": 22322429, + "total_output_tokens": 679670, + "total_tokens": 23002099, + "total_turns": 1567, + "avg_agent_execution_time": 418.525, + "avg_input_tokens": 265743.2024, + "avg_output_tokens": 8091.3095, + "avg_total_tokens": 273834.5119, + "avg_turns": 18.6548, + "per_run_input_tokens": 5580607.25, + "per_run_output_tokens": 169917.5, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus-thinking", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.4167, + "std": 0.0781 + }, + "pass@4": 0.619, + "pass^4": 0.1905 + }, "deepseek-chat": { "total_tasks": 21, "total_agent_execution_time": 29866.240534305573, @@ -3677,29 +4212,6 @@ "pass@4": 0.0476, "pass^4": 0.0476 }, - "claude-opus-4-1": { - "total_tasks": 21, - "total_agent_execution_time": 10822.950607061386, - "total_input_tokens": 5474339, - "total_output_tokens": 205732, - "total_tokens": 5680071, - "total_turns": 522, - "avg_agent_execution_time": 515.3786, - "avg_input_tokens": 260682.8095, - "avg_output_tokens": 9796.7619, - "avg_total_tokens": 270479.5714, - "avg_turns": 24.8571, - "per_run_input_tokens": 5474339.0, - "per_run_output_tokens": 205732.0, - "per_run_cost": 97.544985, - "actual_model_name": "claude-opus-4-1-20250805", - "is_open_source_model": false, - "is_reasoning_model": false, - "pass@1": { - "avg": 0.3333, - "std": 0.0 - } - }, "glm-4-5": { "total_tasks": 21, "total_agent_execution_time": 13283.361280679703, @@ -3750,6 +4262,31 @@ "pass@4": 0.6667, "pass^4": 0.1429 }, + "claude-sonnet-4-5": { + "total_tasks": 21, + "total_agent_execution_time": 20314.365948438644, + "total_input_tokens": 40592706, + "total_output_tokens": 887731, + "total_tokens": 41480437, + "total_turns": 2135, + "avg_agent_execution_time": 241.8377, + "avg_input_tokens": 483246.5, + "avg_output_tokens": 10568.2262, + "avg_total_tokens": 493814.7262, + "avg_turns": 25.4167, + "per_run_input_tokens": 10148176.5, + "per_run_output_tokens": 221932.75, + "per_run_cost": null, + "actual_model_name": "claude-sonnet-4-5-20250929", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.5, + "std": 0.0412 + }, + "pass@4": 0.6667, + "pass^4": 0.381 + }, "claude-sonnet-4-low": { "total_tasks": 21, "total_agent_execution_time": 15643.686558961868, @@ -3875,6 +4412,31 @@ "pass@4": 0.1905, "pass^4": 0.0 }, + "glm-4-6": { + "total_tasks": 21, + "total_agent_execution_time": 22098.47031378746, + "total_input_tokens": 30302473, + "total_output_tokens": 443909, + "total_tokens": 30746382, + "total_turns": 2085, + "avg_agent_execution_time": 263.077, + "avg_input_tokens": 360743.7262, + "avg_output_tokens": 5284.631, + "avg_total_tokens": 366028.3571, + "avg_turns": 24.8214, + "per_run_input_tokens": 7575618.25, + "per_run_output_tokens": 110977.25, + "per_run_cost": null, + "actual_model_name": "", + "is_open_source_model": false, + "is_reasoning_model": false, + "pass@1": { + "avg": 0.5119, + "std": 0.0206 + }, + "pass@4": 0.7143, + "pass^4": 0.2857 + }, "gpt-5-medium": { "total_tasks": 21, "total_agent_execution_time": 28409.77977871895, @@ -4100,6 +4662,31 @@ "pass@4": 0.2857, "pass^4": 0.0 }, + "deepseek-v3-1-terminus": { + "total_tasks": 21, + "total_agent_execution_time": 20219.888365983963, + "total_input_tokens": 24325657, + "total_output_tokens": 295084, + "total_tokens": 24620741, + "total_turns": 1652, + "avg_agent_execution_time": 240.713, + "avg_input_tokens": 289591.1548, + "avg_output_tokens": 3512.9048, + "avg_total_tokens": 293104.0595, + "avg_turns": 19.6667, + "per_run_input_tokens": 6081414.25, + "per_run_output_tokens": 73771.0, + "per_run_cost": null, + "actual_model_name": "deepseek-v3.1-terminus", + "is_open_source_model": true, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.3333, + "std": 0.1992 + }, + "pass@4": 0.5714, + "pass^4": 0.0 + }, "qwen-3-max": { "total_tasks": 21, "total_agent_execution_time": 13406.892842531204, @@ -4300,6 +4887,31 @@ "pass@4": 0.2381, "pass^4": 0.0 }, + "grok-4-fast": { + "total_tasks": 21, + "total_agent_execution_time": 6027.847983598709, + "total_input_tokens": 18555081, + "total_output_tokens": 559540, + "total_tokens": 19114621, + "total_turns": 1559, + "avg_agent_execution_time": 71.7601, + "avg_input_tokens": 220893.8214, + "avg_output_tokens": 6661.1905, + "avg_total_tokens": 227555.0119, + "avg_turns": 18.5595, + "per_run_input_tokens": 4638770.25, + "per_run_output_tokens": 139885.0, + "per_run_cost": null, + "actual_model_name": "grok-4-fast-reasoning", + "is_open_source_model": false, + "is_reasoning_model": true, + "pass@1": { + "avg": 0.5238, + "std": 0.0891 + }, + "pass@4": 0.8095, + "pass^4": 0.2857 + }, "o4-mini": { "total_tasks": 21, "total_agent_execution_time": 7117.457372426987, @@ -4351,5 +4963,6 @@ "pass^4": 0.0476 } }, - "experiment_name": "mcpmark-release" + "experiment_name": "mcpmark-release", + "task_set": "standard" } \ No newline at end of file diff --git a/task_results/build_your_own_x__find_commit_date.json b/task_results/build_your_own_x__find_commit_date.json index 8faa6f3a..2e2cab26 100644 --- a/task_results/build_your_own_x__find_commit_date.json +++ b/task_results/build_your_own_x__find_commit_date.json @@ -2,6 +2,73 @@ "task": "build_your_own_x__find_commit_date", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 357.807208776474, + "agent_execution_time": 357.807208776474, + "token_usage": { + "input_tokens": 1602096, + "output_tokens": 2324, + "total_tokens": 1604420, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + { + "run": "run-2", + "success": false, + "execution_time": 775.9163918495178, + "agent_execution_time": 775.9163918495178, + "token_usage": { + "input_tokens": 295890, + "output_tokens": 19418, + "total_tokens": 315308, + "reasoning_tokens": 18494 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 680.9810349941254, + "agent_execution_time": 680.9810349941254, + "token_usage": { + "input_tokens": 232305, + "output_tokens": 14071, + "total_tokens": 246376, + "reasoning_tokens": 13575 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 940.7171416282654, + "agent_execution_time": 940.7171416282654, + "token_usage": { + "input_tokens": 365629, + "output_tokens": 24970, + "total_tokens": 390599, + "reasoning_tokens": 24116 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 688.86, + "avg_input_tokens": 623980.0, + "avg_output_tokens": 15195.8, + "avg_total_tokens": 639175.8, + "avg_turn_count": 16.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 101.42097783088684, - "agent_execution_time": 101.42097783088684, - "token_usage": { - "input_tokens": 258338, - "output_tokens": 680, - "total_tokens": 259018, - "reasoning_tokens": null - }, - "turn_count": 5 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 101.42, - "avg_input_tokens": 258338.0, - "avg_output_tokens": 680.0, - "avg_total_tokens": 259018.0, - "avg_turn_count": 5.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 9.048184156417847, + "agent_execution_time": 9.048184156417847, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-2", + "success": false, + "execution_time": 152.6909122467041, + "agent_execution_time": 152.6909122467041, + "token_usage": { + "input_tokens": 2193967, + "output_tokens": 2995, + "total_tokens": 2196962, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": false, + "execution_time": 147.16514229774475, + "agent_execution_time": 147.16514229774475, + "token_usage": { + "input_tokens": 2050152, + "output_tokens": 2952, + "total_tokens": 2053104, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": true, + "execution_time": 73.4245445728302, + "agent_execution_time": 73.4245445728302, + "token_usage": { + "input_tokens": 586966, + "output_tokens": 1706, + "total_tokens": 588672, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 95.58, + "avg_input_tokens": 1207771.2, + "avg_output_tokens": 1913.2, + "avg_total_tokens": 1209684.5, + "avg_turn_count": 12.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 533.7350254058838, + "agent_execution_time": 533.7350254058838, + "token_usage": { + "input_tokens": 3259116, + "output_tokens": 3047, + "total_tokens": 3262163, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + { + "run": "run-2", + "success": false, + "execution_time": 243.50679802894592, + "agent_execution_time": 243.50679802894592, + "token_usage": { + "input_tokens": 1265445, + "output_tokens": 1098, + "total_tokens": 1266543, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 184.39719676971436, + "agent_execution_time": 184.39719676971436, + "token_usage": { + "input_tokens": 1070156, + "output_tokens": 515, + "total_tokens": 1070671, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 252.76967668533325, + "agent_execution_time": 252.76967668533325, + "token_usage": { + "input_tokens": 1769908, + "output_tokens": 1489, + "total_tokens": 1771397, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 303.6, + "avg_input_tokens": 1841156.2, + "avg_output_tokens": 1537.2, + "avg_total_tokens": 1842693.5, + "avg_turn_count": 21.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 388.3499140739441, + "agent_execution_time": 388.3499140739441, + "token_usage": { + "input_tokens": 1435464, + "output_tokens": 2579, + "total_tokens": 1438043, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-2", + "success": false, + "execution_time": 12.15988540649414, + "agent_execution_time": 12.15988540649414, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 12.474773406982422, + "agent_execution_time": 12.474773406982422, + "token_usage": { + "input_tokens": 23533, + "output_tokens": 95, + "total_tokens": 23628, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 518.7657308578491, + "agent_execution_time": 518.7657308578491, + "token_usage": { + "input_tokens": 1087397, + "output_tokens": 1637, + "total_tokens": 1089034, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 232.94, + "avg_input_tokens": 636598.5, + "avg_output_tokens": 1077.8, + "avg_total_tokens": 637676.2, + "avg_turn_count": 14.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 2173.348979949951, + "agent_execution_time": 2173.348979949951, + "token_usage": { + "input_tokens": 10285265, + "output_tokens": 128617, + "total_tokens": 10413882, + "reasoning_tokens": 7454 + }, + "turn_count": 40 + }, + { + "run": "run-2", + "success": false, + "execution_time": 82.50000667572021, + "agent_execution_time": 82.50000667572021, + "token_usage": { + "input_tokens": 145342, + "output_tokens": 7883, + "total_tokens": 153225, + "reasoning_tokens": 2148 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": true, + "execution_time": 130.41381669044495, + "agent_execution_time": 130.41381669044495, + "token_usage": { + "input_tokens": 1375606, + "output_tokens": 6358, + "total_tokens": 1381964, + "reasoning_tokens": 5206 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1340.6879160404205, + "agent_execution_time": 1340.6879160404205, + "token_usage": { + "input_tokens": 1283786, + "output_tokens": 120609, + "total_tokens": 1404395, + "reasoning_tokens": 2479 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 931.74, + "avg_input_tokens": 3272499.8, + "avg_output_tokens": 65866.8, + "avg_total_tokens": 3338366.5, + "avg_turn_count": 21.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/build_your_own_x__find_rag_commit.json b/task_results/build_your_own_x__find_rag_commit.json index a4e04b17..960be97c 100644 --- a/task_results/build_your_own_x__find_rag_commit.json +++ b/task_results/build_your_own_x__find_rag_commit.json @@ -2,6 +2,73 @@ "task": "build_your_own_x__find_rag_commit", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 260.5582346916199, + "agent_execution_time": 260.5582346916199, + "token_usage": { + "input_tokens": 819475, + "output_tokens": 1957, + "total_tokens": 821432, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": true, + "execution_time": 1811.415769815445, + "agent_execution_time": 1811.415769815445, + "token_usage": { + "input_tokens": 823100, + "output_tokens": 44703, + "total_tokens": 867803, + "reasoning_tokens": 43139 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1345.8292202949524, + "agent_execution_time": 1345.8292202949524, + "token_usage": { + "input_tokens": 966167, + "output_tokens": 22768, + "total_tokens": 988935, + "reasoning_tokens": 21494 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1041.5495312213898, + "agent_execution_time": 1041.5495312213898, + "token_usage": { + "input_tokens": 562922, + "output_tokens": 27196, + "total_tokens": 590118, + "reasoning_tokens": 26315 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 1114.84, + "avg_input_tokens": 792916.0, + "avg_output_tokens": 24156.0, + "avg_total_tokens": 817072.0, + "avg_turn_count": 19.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 71.28984642028809, - "agent_execution_time": 71.28984642028809, - "token_usage": { - "input_tokens": 100338, - "output_tokens": 599, - "total_tokens": 100937, - "reasoning_tokens": null - }, - "turn_count": 5 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 71.29, - "avg_input_tokens": 100338.0, - "avg_output_tokens": 599.0, - "avg_total_tokens": 100937.0, - "avg_turn_count": 5.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 9.023174524307251, + "agent_execution_time": 9.023174524307251, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-2", + "success": true, + "execution_time": 149.35746383666992, + "agent_execution_time": 149.35746383666992, + "token_usage": { + "input_tokens": 2252545, + "output_tokens": 2737, + "total_tokens": 2255282, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": true, + "execution_time": 170.13444805145264, + "agent_execution_time": 170.13444805145264, + "token_usage": { + "input_tokens": 2508309, + "output_tokens": 3253, + "total_tokens": 2511562, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": true, + "execution_time": 194.2477638721466, + "agent_execution_time": 194.2477638721466, + "token_usage": { + "input_tokens": 2630412, + "output_tokens": 3921, + "total_tokens": 2634333, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 130.69, + "avg_input_tokens": 1847816.5, + "avg_output_tokens": 2477.8, + "avg_total_tokens": 1850294.2, + "avg_turn_count": 17.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 128.51300883293152, + "agent_execution_time": 128.51300883293152, + "token_usage": { + "input_tokens": 1472839, + "output_tokens": 1133, + "total_tokens": 1473972, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 321.92074847221375, + "agent_execution_time": 321.92074847221375, + "token_usage": { + "input_tokens": 1774028, + "output_tokens": 824, + "total_tokens": 1774852, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 173.39985013008118, + "agent_execution_time": 173.39985013008118, + "token_usage": { + "input_tokens": 932115, + "output_tokens": 839, + "total_tokens": 932954, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": true, + "execution_time": 230.04617857933044, + "agent_execution_time": 230.04617857933044, + "token_usage": { + "input_tokens": 1667592, + "output_tokens": 2403, + "total_tokens": 1669995, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 213.47, + "avg_input_tokens": 1461643.5, + "avg_output_tokens": 1299.8, + "avg_total_tokens": 1462943.2, + "avg_turn_count": 17.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 208.98038411140442, + "agent_execution_time": 208.98038411140442, + "token_usage": { + "input_tokens": 537624, + "output_tokens": 1453, + "total_tokens": 539077, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 9.882914543151855, + "agent_execution_time": 9.882914543151855, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 484.1384336948395, + "agent_execution_time": 484.1384336948395, + "token_usage": { + "input_tokens": 856091, + "output_tokens": 1864, + "total_tokens": 857955, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 732.2848217487335, + "agent_execution_time": 732.2848217487335, + "token_usage": { + "input_tokens": 1479523, + "output_tokens": 1166, + "total_tokens": 1480689, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 358.82, + "avg_input_tokens": 718309.5, + "avg_output_tokens": 1120.8, + "avg_total_tokens": 719430.2, + "avg_turn_count": 15.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 77.67906475067139, + "agent_execution_time": 77.67906475067139, + "token_usage": { + "input_tokens": 641675, + "output_tokens": 4859, + "total_tokens": 646534, + "reasoning_tokens": 1672 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 104.13842034339905, + "agent_execution_time": 104.13842034339905, + "token_usage": { + "input_tokens": 459613, + "output_tokens": 7085, + "total_tokens": 466698, + "reasoning_tokens": 2563 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1076.2166833877563, + "agent_execution_time": 1076.2166833877563, + "token_usage": { + "input_tokens": 785061, + "output_tokens": 101054, + "total_tokens": 886115, + "reasoning_tokens": 2377 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 121.34236764907837, + "agent_execution_time": 121.34236764907837, + "token_usage": { + "input_tokens": 818050, + "output_tokens": 7075, + "total_tokens": 825125, + "reasoning_tokens": 5364 + }, + "turn_count": 33 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 344.84, + "avg_input_tokens": 676099.8, + "avg_output_tokens": 30018.2, + "avg_total_tokens": 706118.0, + "avg_turn_count": 19.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/chinook__customer_data_migration.json b/task_results/chinook__customer_data_migration.json index 829f4d5d..6f235cb6 100644 --- a/task_results/chinook__customer_data_migration.json +++ b/task_results/chinook__customer_data_migration.json @@ -2,6 +2,73 @@ "task": "chinook__customer_data_migration", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 226.29387187957764, + "agent_execution_time": 226.29387187957764, + "token_usage": { + "input_tokens": 61388, + "output_tokens": 4272, + "total_tokens": 65660, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 208.74509859085083, + "agent_execution_time": 208.74509859085083, + "token_usage": { + "input_tokens": 61349, + "output_tokens": 4254, + "total_tokens": 65603, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 204.9047417640686, + "agent_execution_time": 204.9047417640686, + "token_usage": { + "input_tokens": 74894, + "output_tokens": 4276, + "total_tokens": 79170, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 444.83052468299866, + "agent_execution_time": 444.83052468299866, + "token_usage": { + "input_tokens": 24824, + "output_tokens": 13065, + "total_tokens": 37889, + "reasoning_tokens": 8912 + }, + "turn_count": 2 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 271.19, + "avg_input_tokens": 55613.8, + "avg_output_tokens": 6466.8, + "avg_total_tokens": 62080.5, + "avg_turn_count": 4.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 1022.2541089057922, - "agent_execution_time": 1022.2541089057922, - "token_usage": { - "input_tokens": 386018, - "output_tokens": 18977, - "total_tokens": 404995, - "reasoning_tokens": null - }, - "turn_count": 19 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 1022.25, - "avg_input_tokens": 386018.0, - "avg_output_tokens": 18977.0, - "avg_total_tokens": 404995.0, - "avg_turn_count": 19.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 22.106448888778687, + "agent_execution_time": 22.106448888778687, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-2", + "success": false, + "execution_time": 3600.1552386283875, + "agent_execution_time": 3600.1552386283875, + "token_usage": { + "input_tokens": 1258126, + "output_tokens": 184097, + "total_tokens": 1442223, + "reasoning_tokens": 0 + }, + "turn_count": 59 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1394.781268119812, + "agent_execution_time": 1394.781268119812, + "token_usage": { + "input_tokens": 2416116, + "output_tokens": 56721, + "total_tokens": 2472837, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1213.39200425148, + "agent_execution_time": 1213.39200425148, + "token_usage": { + "input_tokens": 2354137, + "output_tokens": 44442, + "total_tokens": 2398579, + "reasoning_tokens": 0 + }, + "turn_count": 100 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1557.61, + "avg_input_tokens": 1507094.8, + "avg_output_tokens": 71315.0, + "avg_total_tokens": 1578409.8, + "avg_turn_count": 64.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 139.31443881988525, + "agent_execution_time": 139.31443881988525, + "token_usage": { + "input_tokens": 78384, + "output_tokens": 337, + "total_tokens": 78721, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 150.28711557388306, + "agent_execution_time": 150.28711557388306, + "token_usage": { + "input_tokens": 78208, + "output_tokens": 682, + "total_tokens": 78890, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 158.42241668701172, + "agent_execution_time": 158.42241668701172, + "token_usage": { + "input_tokens": 78015, + "output_tokens": 646, + "total_tokens": 78661, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 163.1177361011505, + "agent_execution_time": 163.1177361011505, + "token_usage": { + "input_tokens": 78451, + "output_tokens": 1880, + "total_tokens": 80331, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 152.79, + "avg_input_tokens": 78264.5, + "avg_output_tokens": 886.2, + "avg_total_tokens": 79150.8, + "avg_turn_count": 6.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 203.7839663028717, + "agent_execution_time": 203.7839663028717, + "token_usage": { + "input_tokens": 61296, + "output_tokens": 4233, + "total_tokens": 65529, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 222.6151738166809, + "agent_execution_time": 222.6151738166809, + "token_usage": { + "input_tokens": 74897, + "output_tokens": 4282, + "total_tokens": 79179, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 205.81352043151855, + "agent_execution_time": 205.81352043151855, + "token_usage": { + "input_tokens": 74958, + "output_tokens": 4297, + "total_tokens": 79255, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.747117757797241, + "agent_execution_time": 8.747117757797241, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 160.24, + "avg_input_tokens": 52787.8, + "avg_output_tokens": 3203.0, + "avg_total_tokens": 55990.8, + "avg_turn_count": 4.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 107.74598097801208, + "agent_execution_time": 107.74598097801208, + "token_usage": { + "input_tokens": 207094, + "output_tokens": 13978, + "total_tokens": 221072, + "reasoning_tokens": 1412 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 100.56838035583496, + "agent_execution_time": 100.56838035583496, + "token_usage": { + "input_tokens": 267760, + "output_tokens": 15491, + "total_tokens": 283251, + "reasoning_tokens": 3363 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 107.42437553405762, + "agent_execution_time": 107.42437553405762, + "token_usage": { + "input_tokens": 101514, + "output_tokens": 14321, + "total_tokens": 115835, + "reasoning_tokens": 1880 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 44.38723421096802, + "agent_execution_time": 44.38723421096802, + "token_usage": { + "input_tokens": 61616, + "output_tokens": 4486, + "total_tokens": 66102, + "reasoning_tokens": 4315 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 90.03, + "avg_input_tokens": 159496.0, + "avg_output_tokens": 12069.0, + "avg_total_tokens": 171565.0, + "avg_turn_count": 9.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/chinook__employee_hierarchy_management.json b/task_results/chinook__employee_hierarchy_management.json index 8614eae9..fc6bb5ae 100644 --- a/task_results/chinook__employee_hierarchy_management.json +++ b/task_results/chinook__employee_hierarchy_management.json @@ -2,6 +2,73 @@ "task": "chinook__employee_hierarchy_management", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 209.23206996917725, + "agent_execution_time": 209.23206996917725, + "token_usage": { + "input_tokens": 245875, + "output_tokens": 2205, + "total_tokens": 248080, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": true, + "execution_time": 204.79339170455933, + "agent_execution_time": 204.79339170455933, + "token_usage": { + "input_tokens": 246121, + "output_tokens": 2232, + "total_tokens": 248353, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": true, + "execution_time": 184.4407684803009, + "agent_execution_time": 184.4407684803009, + "token_usage": { + "input_tokens": 204464, + "output_tokens": 2324, + "total_tokens": 206788, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": true, + "execution_time": 298.06419038772583, + "agent_execution_time": 298.06419038772583, + "token_usage": { + "input_tokens": 13895, + "output_tokens": 8093, + "total_tokens": 21988, + "reasoning_tokens": 5864 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 224.13, + "avg_input_tokens": 177588.8, + "avg_output_tokens": 3713.5, + "avg_total_tokens": 181302.2, + "avg_turn_count": 18.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 1.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 322.3665874004364, - "agent_execution_time": 322.3665874004364, - "token_usage": { - "input_tokens": 110287, - "output_tokens": 5976, - "total_tokens": 116263, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 322.37, - "avg_input_tokens": 110287.0, - "avg_output_tokens": 5976.0, - "avg_total_tokens": 116263.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 118.62710762023926, + "agent_execution_time": 118.62710762023926, + "token_usage": { + "input_tokens": 272418, + "output_tokens": 4058, + "total_tokens": 276476, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": true, + "execution_time": 103.54423666000366, + "agent_execution_time": 103.54423666000366, + "token_usage": { + "input_tokens": 248315, + "output_tokens": 3785, + "total_tokens": 252100, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-3", + "success": true, + "execution_time": 93.01032948493958, + "agent_execution_time": 93.01032948493958, + "token_usage": { + "input_tokens": 168384, + "output_tokens": 4425, + "total_tokens": 172809, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": true, + "execution_time": 137.89153909683228, + "agent_execution_time": 137.89153909683228, + "token_usage": { + "input_tokens": 246161, + "output_tokens": 3990, + "total_tokens": 250151, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 113.27, + "avg_input_tokens": 233819.5, + "avg_output_tokens": 4064.5, + "avg_total_tokens": 237884.0, + "avg_turn_count": 21.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 145.07915711402893, + "agent_execution_time": 145.07915711402893, + "token_usage": { + "input_tokens": 179941, + "output_tokens": 3414, + "total_tokens": 183355, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": true, + "execution_time": 213.0750117301941, + "agent_execution_time": 213.0750117301941, + "token_usage": { + "input_tokens": 205972, + "output_tokens": 3533, + "total_tokens": 209505, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": true, + "execution_time": 284.17301630973816, + "agent_execution_time": 284.17301630973816, + "token_usage": { + "input_tokens": 233220, + "output_tokens": 3776, + "total_tokens": 236996, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": true, + "execution_time": 120.56951808929443, + "agent_execution_time": 120.56951808929443, + "token_usage": { + "input_tokens": 297469, + "output_tokens": 3471, + "total_tokens": 300940, + "reasoning_tokens": 0 + }, + "turn_count": 29 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 190.72, + "avg_input_tokens": 229150.5, + "avg_output_tokens": 3548.5, + "avg_total_tokens": 232699.0, + "avg_turn_count": 23.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 181.4918601512909, + "agent_execution_time": 181.4918601512909, + "token_usage": { + "input_tokens": 185888, + "output_tokens": 2413, + "total_tokens": 188301, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": true, + "execution_time": 201.84227919578552, + "agent_execution_time": 201.84227919578552, + "token_usage": { + "input_tokens": 249784, + "output_tokens": 2340, + "total_tokens": 252124, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": true, + "execution_time": 200.8980212211609, + "agent_execution_time": 200.8980212211609, + "token_usage": { + "input_tokens": 261562, + "output_tokens": 2313, + "total_tokens": 263875, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.158531904220581, + "agent_execution_time": 8.158531904220581, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 148.1, + "avg_input_tokens": 174308.5, + "avg_output_tokens": 1766.5, + "avg_total_tokens": 176075.0, + "avg_turn_count": 18.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 47.389010190963745, + "agent_execution_time": 47.389010190963745, + "token_usage": { + "input_tokens": 122294, + "output_tokens": 3222, + "total_tokens": 125516, + "reasoning_tokens": 1748 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": true, + "execution_time": 46.88154339790344, + "agent_execution_time": 46.88154339790344, + "token_usage": { + "input_tokens": 118297, + "output_tokens": 3777, + "total_tokens": 122074, + "reasoning_tokens": 2361 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": true, + "execution_time": 71.06090593338013, + "agent_execution_time": 71.06090593338013, + "token_usage": { + "input_tokens": 240727, + "output_tokens": 4382, + "total_tokens": 245109, + "reasoning_tokens": 2761 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": true, + "execution_time": 41.21628499031067, + "agent_execution_time": 41.21628499031067, + "token_usage": { + "input_tokens": 101924, + "output_tokens": 4250, + "total_tokens": 106174, + "reasoning_tokens": 3127 + }, + "turn_count": 16 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 51.64, + "avg_input_tokens": 145810.5, + "avg_output_tokens": 3907.8, + "avg_total_tokens": 149718.2, + "avg_turn_count": 21.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/chinook__sales_and_music_charts.json b/task_results/chinook__sales_and_music_charts.json index 2747249e..d96da909 100644 --- a/task_results/chinook__sales_and_music_charts.json +++ b/task_results/chinook__sales_and_music_charts.json @@ -2,6 +2,73 @@ "task": "chinook__sales_and_music_charts", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 339.96108889579773, + "agent_execution_time": 339.96108889579773, + "token_usage": { + "input_tokens": 428898, + "output_tokens": 4524, + "total_tokens": 433422, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-2", + "success": false, + "execution_time": 279.29951310157776, + "agent_execution_time": 279.29951310157776, + "token_usage": { + "input_tokens": 337123, + "output_tokens": 3812, + "total_tokens": 340935, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-3", + "success": false, + "execution_time": 214.09886193275452, + "agent_execution_time": 214.09886193275452, + "token_usage": { + "input_tokens": 216495, + "output_tokens": 2911, + "total_tokens": 219406, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 32.553945541381836, + "agent_execution_time": 32.553945541381836, + "token_usage": { + "input_tokens": 2683, + "output_tokens": 780, + "total_tokens": 3463, + "reasoning_tokens": 749 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 216.48, + "avg_input_tokens": 246299.8, + "avg_output_tokens": 3006.8, + "avg_total_tokens": 249306.5, + "avg_turn_count": 19.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 424.17409896850586, - "agent_execution_time": 424.17409896850586, - "token_usage": { - "input_tokens": 199197, - "output_tokens": 7297, - "total_tokens": 206494, - "reasoning_tokens": null - }, - "turn_count": 29 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 424.17, - "avg_input_tokens": 199197.0, - "avg_output_tokens": 7297.0, - "avg_total_tokens": 206494.0, - "avg_turn_count": 29.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 119.53029990196228, + "agent_execution_time": 119.53029990196228, + "token_usage": { + "input_tokens": 308218, + "output_tokens": 5732, + "total_tokens": 313950, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 101.49867391586304, + "agent_execution_time": 101.49867391586304, + "token_usage": { + "input_tokens": 276512, + "output_tokens": 4542, + "total_tokens": 281054, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": false, + "execution_time": 93.56715822219849, + "agent_execution_time": 93.56715822219849, + "token_usage": { + "input_tokens": 268102, + "output_tokens": 4216, + "total_tokens": 272318, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 101.90559959411621, + "agent_execution_time": 101.90559959411621, + "token_usage": { + "input_tokens": 272276, + "output_tokens": 4433, + "total_tokens": 276709, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 104.13, + "avg_input_tokens": 281277.0, + "avg_output_tokens": 4730.8, + "avg_total_tokens": 286007.8, + "avg_turn_count": 18.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 128.95405292510986, + "agent_execution_time": 128.95405292510986, + "token_usage": { + "input_tokens": 159822, + "output_tokens": 3325, + "total_tokens": 163147, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 318.0042164325714, + "agent_execution_time": 318.0042164325714, + "token_usage": { + "input_tokens": 346957, + "output_tokens": 3706, + "total_tokens": 350663, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-3", + "success": false, + "execution_time": 334.34429025650024, + "agent_execution_time": 334.34429025650024, + "token_usage": { + "input_tokens": 203671, + "output_tokens": 4180, + "total_tokens": 207851, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 122.83406901359558, + "agent_execution_time": 122.83406901359558, + "token_usage": { + "input_tokens": 211858, + "output_tokens": 4209, + "total_tokens": 216067, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 226.03, + "avg_input_tokens": 230577.0, + "avg_output_tokens": 3855.0, + "avg_total_tokens": 234432.0, + "avg_turn_count": 21.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 270.4434940814972, + "agent_execution_time": 270.4434940814972, + "token_usage": { + "input_tokens": 340352, + "output_tokens": 4035, + "total_tokens": 344387, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 255.62756371498108, + "agent_execution_time": 255.62756371498108, + "token_usage": { + "input_tokens": 263974, + "output_tokens": 3607, + "total_tokens": 267581, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 290.57821702957153, + "agent_execution_time": 290.57821702957153, + "token_usage": { + "input_tokens": 390092, + "output_tokens": 3995, + "total_tokens": 394087, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.829689741134644, + "agent_execution_time": 8.829689741134644, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 206.37, + "avg_input_tokens": 248604.5, + "avg_output_tokens": 2909.2, + "avg_total_tokens": 251513.8, + "avg_turn_count": 19.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 52.323203325271606, + "agent_execution_time": 52.323203325271606, + "token_usage": { + "input_tokens": 175397, + "output_tokens": 4181, + "total_tokens": 179578, + "reasoning_tokens": 2682 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 41.52652311325073, + "agent_execution_time": 41.52652311325073, + "token_usage": { + "input_tokens": 112951, + "output_tokens": 4029, + "total_tokens": 116980, + "reasoning_tokens": 2467 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": false, + "execution_time": 32.15245079994202, + "agent_execution_time": 32.15245079994202, + "token_usage": { + "input_tokens": 112270, + "output_tokens": 2676, + "total_tokens": 114946, + "reasoning_tokens": 1130 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 35.22850847244263, + "agent_execution_time": 35.22850847244263, + "token_usage": { + "input_tokens": 66931, + "output_tokens": 5081, + "total_tokens": 72012, + "reasoning_tokens": 3887 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 40.31, + "avg_input_tokens": 116887.2, + "avg_output_tokens": 3991.8, + "avg_total_tokens": 120879.0, + "avg_turn_count": 13.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/claude-code__automated_changelog_generation.json b/task_results/claude-code__automated_changelog_generation.json index 388108a2..c0c3f166 100644 --- a/task_results/claude-code__automated_changelog_generation.json +++ b/task_results/claude-code__automated_changelog_generation.json @@ -2,6 +2,73 @@ "task": "claude-code__automated_changelog_generation", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 328.56732177734375, + "agent_execution_time": 328.56732177734375, + "token_usage": { + "input_tokens": 486413, + "output_tokens": 5640, + "total_tokens": 492053, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 335.2862730026245, + "agent_execution_time": 335.2862730026245, + "token_usage": { + "input_tokens": 193716, + "output_tokens": 7517, + "total_tokens": 201233, + "reasoning_tokens": 3380 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 601.6498417854309, + "agent_execution_time": 601.6498417854309, + "token_usage": { + "input_tokens": 185533, + "output_tokens": 14256, + "total_tokens": 199789, + "reasoning_tokens": 11439 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 69.64990425109863, + "agent_execution_time": 69.64990425109863, + "token_usage": { + "input_tokens": 24123, + "output_tokens": 1865, + "total_tokens": 25988, + "reasoning_tokens": 703 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 333.79, + "avg_input_tokens": 222446.2, + "avg_output_tokens": 7319.5, + "avg_total_tokens": 229765.8, + "avg_turn_count": 5.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 1.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 486.5178973674774, - "agent_execution_time": 486.5178973674774, - "token_usage": { - "input_tokens": 551409, - "output_tokens": 6538, - "total_tokens": 557947, - "reasoning_tokens": null - }, - "turn_count": 10 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 486.52, - "avg_input_tokens": 551409.0, - "avg_output_tokens": 6538.0, - "avg_total_tokens": 557947.0, - "avg_turn_count": 10.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 335.13299441337585, + "agent_execution_time": 335.13299441337585, + "token_usage": { + "input_tokens": 424907, + "output_tokens": 14771, + "total_tokens": 439678, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 295.59840059280396, + "agent_execution_time": 295.59840059280396, + "token_usage": { + "input_tokens": 426113, + "output_tokens": 13573, + "total_tokens": 439686, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 225.89520597457886, + "agent_execution_time": 225.89520597457886, + "token_usage": { + "input_tokens": 475948, + "output_tokens": 9401, + "total_tokens": 485349, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1719.6133711338043, + "agent_execution_time": 1719.6133711338043, + "token_usage": { + "input_tokens": 1846401, + "output_tokens": 78909, + "total_tokens": 1925310, + "reasoning_tokens": 0 + }, + "turn_count": 33 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 644.06, + "avg_input_tokens": 793342.2, + "avg_output_tokens": 29163.5, + "avg_total_tokens": 822505.8, + "avg_turn_count": 14.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 245.26052379608154, + "agent_execution_time": 245.26052379608154, + "token_usage": { + "input_tokens": 344295, + "output_tokens": 6200, + "total_tokens": 350495, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": true, + "execution_time": 204.87113118171692, + "agent_execution_time": 204.87113118171692, + "token_usage": { + "input_tokens": 368764, + "output_tokens": 6340, + "total_tokens": 375104, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": true, + "execution_time": 204.55552554130554, + "agent_execution_time": 204.55552554130554, + "token_usage": { + "input_tokens": 372335, + "output_tokens": 7264, + "total_tokens": 379599, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": true, + "execution_time": 185.73535060882568, + "agent_execution_time": 185.73535060882568, + "token_usage": { + "input_tokens": 335993, + "output_tokens": 6879, + "total_tokens": 342872, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 210.11, + "avg_input_tokens": 355346.8, + "avg_output_tokens": 6670.8, + "avg_total_tokens": 362017.5, + "avg_turn_count": 11.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 473.3854594230652, + "agent_execution_time": 473.3854594230652, + "token_usage": { + "input_tokens": 827315, + "output_tokens": 7174, + "total_tokens": 834489, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.389399766921997, + "agent_execution_time": 10.389399766921997, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 443.25855326652527, + "agent_execution_time": 443.25855326652527, + "token_usage": { + "input_tokens": 537077, + "output_tokens": 5536, + "total_tokens": 542613, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 169.28932094573975, + "agent_execution_time": 169.28932094573975, + "token_usage": { + "input_tokens": 153052, + "output_tokens": 2790, + "total_tokens": 155842, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 274.08, + "avg_input_tokens": 379361.0, + "avg_output_tokens": 3875.0, + "avg_total_tokens": 383236.0, + "avg_turn_count": 7.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 56.097662925720215, + "agent_execution_time": 56.097662925720215, + "token_usage": { + "input_tokens": 140723, + "output_tokens": 4328, + "total_tokens": 145051, + "reasoning_tokens": 2169 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": false, + "execution_time": 49.320733070373535, + "agent_execution_time": 49.320733070373535, + "token_usage": { + "input_tokens": 180963, + "output_tokens": 4340, + "total_tokens": 185303, + "reasoning_tokens": 2196 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": true, + "execution_time": 46.808067083358765, + "agent_execution_time": 46.808067083358765, + "token_usage": { + "input_tokens": 144449, + "output_tokens": 5246, + "total_tokens": 149695, + "reasoning_tokens": 3258 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": true, + "execution_time": 66.99075412750244, + "agent_execution_time": 66.99075412750244, + "token_usage": { + "input_tokens": 185793, + "output_tokens": 6143, + "total_tokens": 191936, + "reasoning_tokens": 3966 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 54.8, + "avg_input_tokens": 162982.0, + "avg_output_tokens": 5014.2, + "avg_total_tokens": 167996.2, + "avg_turn_count": 4.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/claude-code__claude_collaboration_analysis.json b/task_results/claude-code__claude_collaboration_analysis.json index 42454b9e..14759418 100644 --- a/task_results/claude-code__claude_collaboration_analysis.json +++ b/task_results/claude-code__claude_collaboration_analysis.json @@ -2,6 +2,73 @@ "task": "claude-code__claude_collaboration_analysis", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 93.30665850639343, + "agent_execution_time": 93.30665850639343, + "token_usage": { + "input_tokens": 260947, + "output_tokens": 1143, + "total_tokens": 262090, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 730.9451944828033, + "agent_execution_time": 730.9451944828033, + "token_usage": { + "input_tokens": 224095, + "output_tokens": 19142, + "total_tokens": 243237, + "reasoning_tokens": 18435 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 2150.869268655777, + "agent_execution_time": 2150.869268655777, + "token_usage": { + "input_tokens": 371407, + "output_tokens": 53642, + "total_tokens": 425049, + "reasoning_tokens": 52572 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 568.0564692020416, + "agent_execution_time": 568.0564692020416, + "token_usage": { + "input_tokens": 224089, + "output_tokens": 16239, + "total_tokens": 240328, + "reasoning_tokens": 15616 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 885.79, + "avg_input_tokens": 270134.5, + "avg_output_tokens": 22541.5, + "avg_total_tokens": 292676.0, + "avg_turn_count": 4.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 59.63302779197693, - "agent_execution_time": 59.63302779197693, - "token_usage": { - "input_tokens": 214092, - "output_tokens": 271, - "total_tokens": 214363, - "reasoning_tokens": null - }, - "turn_count": 2 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 59.63, - "avg_input_tokens": 214092.0, - "avg_output_tokens": 271.0, - "avg_total_tokens": 214363.0, - "avg_turn_count": 2.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 80.06109738349915, + "agent_execution_time": 80.06109738349915, + "token_usage": { + "input_tokens": 218564, + "output_tokens": 1587, + "total_tokens": 220151, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": false, + "execution_time": 137.86703252792358, + "agent_execution_time": 137.86703252792358, + "token_usage": { + "input_tokens": 648856, + "output_tokens": 4738, + "total_tokens": 653594, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 56.66862773895264, + "agent_execution_time": 56.66862773895264, + "token_usage": { + "input_tokens": 297445, + "output_tokens": 1765, + "total_tokens": 299210, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 47.77523708343506, + "agent_execution_time": 47.77523708343506, + "token_usage": { + "input_tokens": 225802, + "output_tokens": 1607, + "total_tokens": 227409, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 80.59, + "avg_input_tokens": 347666.8, + "avg_output_tokens": 2424.2, + "avg_total_tokens": 350091.0, + "avg_turn_count": 5.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 62.151270627975464, + "agent_execution_time": 62.151270627975464, + "token_usage": { + "input_tokens": 207183, + "output_tokens": 1169, + "total_tokens": 208352, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 64.97355198860168, + "agent_execution_time": 64.97355198860168, + "token_usage": { + "input_tokens": 207345, + "output_tokens": 1346, + "total_tokens": 208691, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 53.872310638427734, + "agent_execution_time": 53.872310638427734, + "token_usage": { + "input_tokens": 262141, + "output_tokens": 1215, + "total_tokens": 263356, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 53.556227684020996, + "agent_execution_time": 53.556227684020996, + "token_usage": { + "input_tokens": 208060, + "output_tokens": 1763, + "total_tokens": 209823, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 58.64, + "avg_input_tokens": 221182.2, + "avg_output_tokens": 1373.2, + "avg_total_tokens": 222555.5, + "avg_turn_count": 5.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 111.53016495704651, + "agent_execution_time": 111.53016495704651, + "token_usage": { + "input_tokens": 260912, + "output_tokens": 1286, + "total_tokens": 262198, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 11.998449563980103, + "agent_execution_time": 11.998449563980103, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 159.6641800403595, + "agent_execution_time": 159.6641800403595, + "token_usage": { + "input_tokens": 287885, + "output_tokens": 791, + "total_tokens": 288676, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 66.93278241157532, + "agent_execution_time": 66.93278241157532, + "token_usage": { + "input_tokens": 135269, + "output_tokens": 714, + "total_tokens": 135983, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 87.53, + "avg_input_tokens": 171016.5, + "avg_output_tokens": 697.8, + "avg_total_tokens": 171714.2, + "avg_turn_count": 3.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 28.926344871520996, + "agent_execution_time": 28.926344871520996, + "token_usage": { + "input_tokens": 99563, + "output_tokens": 2638, + "total_tokens": 102201, + "reasoning_tokens": 2306 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": false, + "execution_time": 32.524813413619995, + "agent_execution_time": 32.524813413619995, + "token_usage": { + "input_tokens": 163099, + "output_tokens": 2263, + "total_tokens": 165362, + "reasoning_tokens": 1820 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 31.306444883346558, + "agent_execution_time": 31.306444883346558, + "token_usage": { + "input_tokens": 113851, + "output_tokens": 2170, + "total_tokens": 116021, + "reasoning_tokens": 1810 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 43.91437554359436, + "agent_execution_time": 43.91437554359436, + "token_usage": { + "input_tokens": 149866, + "output_tokens": 2950, + "total_tokens": 152816, + "reasoning_tokens": 2571 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 34.17, + "avg_input_tokens": 131594.8, + "avg_output_tokens": 2505.2, + "avg_total_tokens": 134100.0, + "avg_turn_count": 5.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/claude-code__critical_issue_hotfix_workflow.json b/task_results/claude-code__critical_issue_hotfix_workflow.json index 9f15ffeb..8bfa2ad9 100644 --- a/task_results/claude-code__critical_issue_hotfix_workflow.json +++ b/task_results/claude-code__critical_issue_hotfix_workflow.json @@ -2,6 +2,73 @@ "task": "claude-code__critical_issue_hotfix_workflow", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 177.06695079803467, + "agent_execution_time": 177.06695079803467, + "token_usage": { + "input_tokens": 220752, + "output_tokens": 2584, + "total_tokens": 223336, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": true, + "execution_time": 634.8860394954681, + "agent_execution_time": 634.8860394954681, + "token_usage": { + "input_tokens": 141976, + "output_tokens": 16958, + "total_tokens": 158934, + "reasoning_tokens": 15301 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": true, + "execution_time": 701.3802073001862, + "agent_execution_time": 701.3802073001862, + "token_usage": { + "input_tokens": 168780, + "output_tokens": 17235, + "total_tokens": 186015, + "reasoning_tokens": 15356 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 362.0613806247711, + "agent_execution_time": 362.0613806247711, + "token_usage": { + "input_tokens": 79970, + "output_tokens": 10124, + "total_tokens": 90094, + "reasoning_tokens": 8270 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 468.85, + "avg_input_tokens": 152869.5, + "avg_output_tokens": 11725.2, + "avg_total_tokens": 164594.8, + "avg_turn_count": 6.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 325.1222264766693, - "agent_execution_time": 325.1222264766693, - "token_usage": { - "input_tokens": 355340, - "output_tokens": 4063, - "total_tokens": 359403, - "reasoning_tokens": null - }, - "turn_count": 11 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 325.12, - "avg_input_tokens": 355340.0, - "avg_output_tokens": 4063.0, - "avg_total_tokens": 359403.0, - "avg_turn_count": 11.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 173.35112285614014, + "agent_execution_time": 173.35112285614014, + "token_usage": { + "input_tokens": 341396, + "output_tokens": 4934, + "total_tokens": 346330, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": true, + "execution_time": 136.18039441108704, + "agent_execution_time": 136.18039441108704, + "token_usage": { + "input_tokens": 393881, + "output_tokens": 4392, + "total_tokens": 398273, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": true, + "execution_time": 126.43626260757446, + "agent_execution_time": 126.43626260757446, + "token_usage": { + "input_tokens": 301671, + "output_tokens": 4326, + "total_tokens": 305997, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": true, + "execution_time": 117.63972783088684, + "agent_execution_time": 117.63972783088684, + "token_usage": { + "input_tokens": 250105, + "output_tokens": 4189, + "total_tokens": 254294, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 138.4, + "avg_input_tokens": 321763.2, + "avg_output_tokens": 4460.2, + "avg_total_tokens": 326223.5, + "avg_turn_count": 10.25, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 111.63058614730835, + "agent_execution_time": 111.63058614730835, + "token_usage": { + "input_tokens": 163919, + "output_tokens": 2634, + "total_tokens": 166553, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": true, + "execution_time": 122.18994617462158, + "agent_execution_time": 122.18994617462158, + "token_usage": { + "input_tokens": 164978, + "output_tokens": 2915, + "total_tokens": 167893, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": true, + "execution_time": 73.52546429634094, + "agent_execution_time": 73.52546429634094, + "token_usage": { + "input_tokens": 161348, + "output_tokens": 1991, + "total_tokens": 163339, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 102.83816957473755, + "agent_execution_time": 102.83816957473755, + "token_usage": { + "input_tokens": 305582, + "output_tokens": 4094, + "total_tokens": 309676, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 102.55, + "avg_input_tokens": 198956.8, + "avg_output_tokens": 2908.5, + "avg_total_tokens": 201865.2, + "avg_turn_count": 11.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 189.12998604774475, + "agent_execution_time": 189.12998604774475, + "token_usage": { + "input_tokens": 220970, + "output_tokens": 2647, + "total_tokens": 223617, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.897485256195068, + "agent_execution_time": 10.897485256195068, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 278.516259431839, + "agent_execution_time": 278.516259431839, + "token_usage": { + "input_tokens": 300591, + "output_tokens": 2969, + "total_tokens": 303560, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 17.21670365333557, + "agent_execution_time": 17.21670365333557, + "token_usage": { + "input_tokens": 24431, + "output_tokens": 276, + "total_tokens": 24707, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 123.94, + "avg_input_tokens": 136498.0, + "avg_output_tokens": 1473.0, + "avg_total_tokens": 137971.0, + "avg_turn_count": 4.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 53.511555194854736, + "agent_execution_time": 53.511555194854736, + "token_usage": { + "input_tokens": 73729, + "output_tokens": 4000, + "total_tokens": 77729, + "reasoning_tokens": 1834 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": false, + "execution_time": 47.90933275222778, + "agent_execution_time": 47.90933275222778, + "token_usage": { + "input_tokens": 218094, + "output_tokens": 3410, + "total_tokens": 221504, + "reasoning_tokens": 1883 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 72.52073359489441, + "agent_execution_time": 72.52073359489441, + "token_usage": { + "input_tokens": 292221, + "output_tokens": 4537, + "total_tokens": 296758, + "reasoning_tokens": 2442 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 50.27313423156738, + "agent_execution_time": 50.27313423156738, + "token_usage": { + "input_tokens": 216188, + "output_tokens": 3135, + "total_tokens": 219323, + "reasoning_tokens": 1475 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 56.05, + "avg_input_tokens": 200058.0, + "avg_output_tokens": 3770.5, + "avg_total_tokens": 203828.5, + "avg_turn_count": 10.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/claude-code__feature_commit_tracking.json b/task_results/claude-code__feature_commit_tracking.json index 8c0b7a98..63058b35 100644 --- a/task_results/claude-code__feature_commit_tracking.json +++ b/task_results/claude-code__feature_commit_tracking.json @@ -2,6 +2,73 @@ "task": "claude-code__feature_commit_tracking", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 287.90511178970337, + "agent_execution_time": 287.90511178970337, + "token_usage": { + "input_tokens": 1373753, + "output_tokens": 2258, + "total_tokens": 1376011, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": false, + "execution_time": 871.6150290966034, + "agent_execution_time": 871.6150290966034, + "token_usage": { + "input_tokens": 189344, + "output_tokens": 18399, + "total_tokens": 207743, + "reasoning_tokens": 17934 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1428.207557439804, + "agent_execution_time": 1428.207557439804, + "token_usage": { + "input_tokens": 805277, + "output_tokens": 27661, + "total_tokens": 832938, + "reasoning_tokens": 26100 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 522.0259490013123, + "agent_execution_time": 522.0259490013123, + "token_usage": { + "input_tokens": 275952, + "output_tokens": 12773, + "total_tokens": 288725, + "reasoning_tokens": 12405 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 777.44, + "avg_input_tokens": 661081.5, + "avg_output_tokens": 15272.8, + "avg_total_tokens": 676354.2, + "avg_turn_count": 12.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 78.38062977790833, - "agent_execution_time": 78.38062977790833, - "token_usage": { - "input_tokens": 243270, - "output_tokens": 492, - "total_tokens": 243762, - "reasoning_tokens": null - }, - "turn_count": 4 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 78.38, - "avg_input_tokens": 243270.0, - "avg_output_tokens": 492.0, - "avg_total_tokens": 243762.0, - "avg_turn_count": 4.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 120.03905081748962, + "agent_execution_time": 120.03905081748962, + "token_usage": { + "input_tokens": 745728, + "output_tokens": 2635, + "total_tokens": 748363, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 114.45466208457947, + "agent_execution_time": 114.45466208457947, + "token_usage": { + "input_tokens": 1330661, + "output_tokens": 2650, + "total_tokens": 1333311, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 154.2811348438263, + "agent_execution_time": 154.2811348438263, + "token_usage": { + "input_tokens": 1274078, + "output_tokens": 3184, + "total_tokens": 1277262, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 91.4626739025116, + "agent_execution_time": 91.4626739025116, + "token_usage": { + "input_tokens": 893881, + "output_tokens": 1977, + "total_tokens": 895858, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 120.06, + "avg_input_tokens": 1061087.0, + "avg_output_tokens": 2611.5, + "avg_total_tokens": 1063698.5, + "avg_turn_count": 11.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 180.24797582626343, + "agent_execution_time": 180.24797582626343, + "token_usage": { + "input_tokens": 949135, + "output_tokens": 844, + "total_tokens": 949979, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 296.18235182762146, + "agent_execution_time": 296.18235182762146, + "token_usage": { + "input_tokens": 1394980, + "output_tokens": 1863, + "total_tokens": 1396843, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 203.18395590782166, + "agent_execution_time": 203.18395590782166, + "token_usage": { + "input_tokens": 1098602, + "output_tokens": 1277, + "total_tokens": 1099879, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 161.69209122657776, + "agent_execution_time": 161.69209122657776, + "token_usage": { + "input_tokens": 1619263, + "output_tokens": 3772, + "total_tokens": 1623035, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 210.33, + "avg_input_tokens": 1265495.0, + "avg_output_tokens": 1939.0, + "avg_total_tokens": 1267434.0, + "avg_turn_count": 17.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 326.2840278148651, + "agent_execution_time": 326.2840278148651, + "token_usage": { + "input_tokens": 1671658, + "output_tokens": 2548, + "total_tokens": 1674206, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.507361650466919, + "agent_execution_time": 10.507361650466919, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 991.8688023090363, + "agent_execution_time": 991.8688023090363, + "token_usage": { + "input_tokens": 1484538, + "output_tokens": 1838, + "total_tokens": 1486376, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 805.1356992721558, + "agent_execution_time": 805.1356992721558, + "token_usage": { + "input_tokens": 1697275, + "output_tokens": 2297, + "total_tokens": 1699572, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 533.45, + "avg_input_tokens": 1213367.8, + "avg_output_tokens": 1670.8, + "avg_total_tokens": 1215038.5, + "avg_turn_count": 16.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 76.72193765640259, + "agent_execution_time": 76.72193765640259, + "token_usage": { + "input_tokens": 512304, + "output_tokens": 6269, + "total_tokens": 518573, + "reasoning_tokens": 3420 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 117.03881025314331, + "agent_execution_time": 117.03881025314331, + "token_usage": { + "input_tokens": 1127110, + "output_tokens": 6935, + "total_tokens": 1134045, + "reasoning_tokens": 3869 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": false, + "execution_time": 88.72263360023499, + "agent_execution_time": 88.72263360023499, + "token_usage": { + "input_tokens": 548929, + "output_tokens": 5501, + "total_tokens": 554430, + "reasoning_tokens": 2861 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 64.88823890686035, + "agent_execution_time": 64.88823890686035, + "token_usage": { + "input_tokens": 444895, + "output_tokens": 4457, + "total_tokens": 449352, + "reasoning_tokens": 2426 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 86.84, + "avg_input_tokens": 658309.5, + "avg_output_tokens": 5790.5, + "avg_total_tokens": 664100.0, + "avg_turn_count": 18.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/claude-code__label_color_standardization.json b/task_results/claude-code__label_color_standardization.json index f5ca5d76..16a559a3 100644 --- a/task_results/claude-code__label_color_standardization.json +++ b/task_results/claude-code__label_color_standardization.json @@ -2,6 +2,68 @@ "task": "claude-code__label_color_standardization", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 30.060205221176147, + "agent_execution_time": 30.060205221176147, + "token_usage": {}, + "turn_count": 0 + }, + { + "run": "run-2", + "success": false, + "execution_time": 272.34059476852417, + "agent_execution_time": 272.34059476852417, + "token_usage": { + "input_tokens": 24064, + "output_tokens": 7793, + "total_tokens": 31857, + "reasoning_tokens": 7728 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1245.5912964344025, + "agent_execution_time": 1245.5912964344025, + "token_usage": { + "input_tokens": 383667, + "output_tokens": 26100, + "total_tokens": 409767, + "reasoning_tokens": 24258 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 165.86075901985168, + "agent_execution_time": 165.86075901985168, + "token_usage": { + "input_tokens": 24064, + "output_tokens": 4198, + "total_tokens": 28262, + "reasoning_tokens": 4127 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 428.46, + "avg_input_tokens": 107948.8, + "avg_output_tokens": 9522.8, + "avg_total_tokens": 117471.5, + "avg_turn_count": 2.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +198,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 337.7997958660126, - "agent_execution_time": 337.7997958660126, - "token_usage": { - "input_tokens": 651976, - "output_tokens": 3885, - "total_tokens": 655861, - "reasoning_tokens": null - }, - "turn_count": 13 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 337.8, - "avg_input_tokens": 651976.0, - "avg_output_tokens": 3885.0, - "avg_total_tokens": 655861.0, - "avg_turn_count": 13.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +332,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 185.12812423706055, + "agent_execution_time": 185.12812423706055, + "token_usage": { + "input_tokens": 859308, + "output_tokens": 4642, + "total_tokens": 863950, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 186.4637885093689, + "agent_execution_time": 186.4637885093689, + "token_usage": { + "input_tokens": 1680481, + "output_tokens": 5281, + "total_tokens": 1685762, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 170.27613186836243, + "agent_execution_time": 170.27613186836243, + "token_usage": { + "input_tokens": 872897, + "output_tokens": 5731, + "total_tokens": 878628, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": true, + "execution_time": 167.31954193115234, + "agent_execution_time": 167.31954193115234, + "token_usage": { + "input_tokens": 1349621, + "output_tokens": 4746, + "total_tokens": 1354367, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 177.3, + "avg_input_tokens": 1190576.8, + "avg_output_tokens": 5100.0, + "avg_total_tokens": 1195676.8, + "avg_turn_count": 12.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +734,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 105.3499207496643, + "agent_execution_time": 105.3499207496643, + "token_usage": { + "input_tokens": 108063, + "output_tokens": 2462, + "total_tokens": 110525, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 114.19231700897217, + "agent_execution_time": 114.19231700897217, + "token_usage": { + "input_tokens": 129718, + "output_tokens": 2504, + "total_tokens": 132222, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 100.34483337402344, + "agent_execution_time": 100.34483337402344, + "token_usage": { + "input_tokens": 144454, + "output_tokens": 2860, + "total_tokens": 147314, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 105.05418539047241, + "agent_execution_time": 105.05418539047241, + "token_usage": { + "input_tokens": 100743, + "output_tokens": 3466, + "total_tokens": 104209, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 106.24, + "avg_input_tokens": 120744.5, + "avg_output_tokens": 2823.0, + "avg_total_tokens": 123567.5, + "avg_turn_count": 9.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1404,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 70.51958465576172, + "agent_execution_time": 70.51958465576172, + "token_usage": { + "input_tokens": 148346, + "output_tokens": 253, + "total_tokens": 148599, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.07112979888916, + "agent_execution_time": 10.07112979888916, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 103.45589923858643, + "agent_execution_time": 103.45589923858643, + "token_usage": { + "input_tokens": 24064, + "output_tokens": 2264, + "total_tokens": 26328, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 316.0364558696747, + "agent_execution_time": 316.0364558696747, + "token_usage": { + "input_tokens": 575619, + "output_tokens": 2715, + "total_tokens": 578334, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 125.02, + "avg_input_tokens": 187007.2, + "avg_output_tokens": 1308.0, + "avg_total_tokens": 188315.2, + "avg_turn_count": 6.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2007,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 101.20335292816162, + "agent_execution_time": 101.20335292816162, + "token_usage": { + "input_tokens": 335088, + "output_tokens": 7304, + "total_tokens": 342392, + "reasoning_tokens": 4015 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 373.88942766189575, + "agent_execution_time": 373.88942766189575, + "token_usage": { + "input_tokens": 442298, + "output_tokens": 28200, + "total_tokens": 470498, + "reasoning_tokens": 6453 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 84.03827977180481, + "agent_execution_time": 84.03827977180481, + "token_usage": { + "input_tokens": 831597, + "output_tokens": 6154, + "total_tokens": 837751, + "reasoning_tokens": 4240 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 113.16027569770813, + "agent_execution_time": 113.16027569770813, + "token_usage": { + "input_tokens": 747581, + "output_tokens": 6563, + "total_tokens": 754144, + "reasoning_tokens": 4657 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 168.07, + "avg_input_tokens": 589141.0, + "avg_output_tokens": 12055.2, + "avg_total_tokens": 601196.2, + "avg_turn_count": 10.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/company_in_a_box__employee_onboarding.json b/task_results/company_in_a_box__employee_onboarding.json index ba681375..82a8d351 100644 --- a/task_results/company_in_a_box__employee_onboarding.json +++ b/task_results/company_in_a_box__employee_onboarding.json @@ -2,6 +2,73 @@ "task": "company_in_a_box__employee_onboarding", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 334.27523612976074, + "agent_execution_time": 334.27523612976074, + "token_usage": { + "input_tokens": 776693, + "output_tokens": 4201, + "total_tokens": 780894, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1877.3445603847504, + "agent_execution_time": 1877.3445603847504, + "token_usage": { + "input_tokens": 166115, + "output_tokens": 42539, + "total_tokens": 208654, + "reasoning_tokens": 41318 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 371.6750223636627, + "agent_execution_time": 371.6750223636627, + "token_usage": { + "input_tokens": 827869, + "output_tokens": 4746, + "total_tokens": 832615, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-4", + "success": false, + "execution_time": 293.0011365413666, + "agent_execution_time": 293.0011365413666, + "token_usage": { + "input_tokens": 456566, + "output_tokens": 3925, + "total_tokens": 460491, + "reasoning_tokens": 230 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 719.07, + "avg_input_tokens": 556810.8, + "avg_output_tokens": 13852.8, + "avg_total_tokens": 570663.5, + "avg_turn_count": 22.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 482.5655288696289, - "agent_execution_time": 482.5655288696289, - "token_usage": { - "input_tokens": 400292, - "output_tokens": 7548, - "total_tokens": 407840, - "reasoning_tokens": null - }, - "turn_count": 15 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 482.57, - "avg_input_tokens": 400292.0, - "avg_output_tokens": 7548.0, - "avg_total_tokens": 407840.0, - "avg_turn_count": 15.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 7.384501695632935, + "agent_execution_time": 7.384501695632935, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-2", + "success": false, + "execution_time": 177.6273341178894, + "agent_execution_time": 177.6273341178894, + "token_usage": { + "input_tokens": 548642, + "output_tokens": 8107, + "total_tokens": 556749, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 202.14041090011597, + "agent_execution_time": 202.14041090011597, + "token_usage": { + "input_tokens": 1340614, + "output_tokens": 6531, + "total_tokens": 1347145, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 199.804523229599, + "agent_execution_time": 199.804523229599, + "token_usage": { + "input_tokens": 1019675, + "output_tokens": 6257, + "total_tokens": 1025932, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 146.74, + "avg_input_tokens": 727232.8, + "avg_output_tokens": 5223.8, + "avg_total_tokens": 732456.5, + "avg_turn_count": 15.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 1406.123925447464, + "agent_execution_time": 1406.123925447464, + "token_usage": { + "input_tokens": 6852872, + "output_tokens": 11407, + "total_tokens": 6864279, + "reasoning_tokens": 0 + }, + "turn_count": 79 + }, + { + "run": "run-2", + "success": false, + "execution_time": 146.4455714225769, + "agent_execution_time": 146.4455714225769, + "token_usage": { + "input_tokens": 313058, + "output_tokens": 3139, + "total_tokens": 316197, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 329.0322012901306, + "agent_execution_time": 329.0322012901306, + "token_usage": { + "input_tokens": 202914, + "output_tokens": 6891, + "total_tokens": 209805, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 258.61174607276917, + "agent_execution_time": 258.61174607276917, + "token_usage": { + "input_tokens": 122120, + "output_tokens": 1653, + "total_tokens": 123773, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 535.05, + "avg_input_tokens": 1872741.0, + "avg_output_tokens": 5772.5, + "avg_total_tokens": 1878513.5, + "avg_turn_count": 32.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 419.08600211143494, + "agent_execution_time": 419.08600211143494, + "token_usage": { + "input_tokens": 1559484, + "output_tokens": 4581, + "total_tokens": 1564065, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": true, + "execution_time": 220.81275343894958, + "agent_execution_time": 220.81275343894958, + "token_usage": { + "input_tokens": 284282, + "output_tokens": 3970, + "total_tokens": 288252, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 100.90077924728394, + "agent_execution_time": 100.90077924728394, + "token_usage": { + "input_tokens": 198295, + "output_tokens": 705, + "total_tokens": 199000, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 346.02288794517517, + "agent_execution_time": 346.02288794517517, + "token_usage": { + "input_tokens": 670581, + "output_tokens": 4181, + "total_tokens": 674762, + "reasoning_tokens": 0 + }, + "turn_count": 28 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 271.71, + "avg_input_tokens": 678160.5, + "avg_output_tokens": 3359.2, + "avg_total_tokens": 681519.8, + "avg_turn_count": 19.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 152.7811119556427, + "agent_execution_time": 152.7811119556427, + "token_usage": { + "input_tokens": 370290, + "output_tokens": 10484, + "total_tokens": 380774, + "reasoning_tokens": 7985 + }, + "turn_count": 30 + }, + { + "run": "run-2", + "success": false, + "execution_time": 82.09955382347107, + "agent_execution_time": 82.09955382347107, + "token_usage": { + "input_tokens": 232701, + "output_tokens": 7011, + "total_tokens": 239712, + "reasoning_tokens": 5431 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 100.55681276321411, + "agent_execution_time": 100.55681276321411, + "token_usage": { + "input_tokens": 251647, + "output_tokens": 9730, + "total_tokens": 261377, + "reasoning_tokens": 3990 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 111.66854643821716, + "agent_execution_time": 111.66854643821716, + "token_usage": { + "input_tokens": 213459, + "output_tokens": 9190, + "total_tokens": 222649, + "reasoning_tokens": 7244 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 111.78, + "avg_input_tokens": 267024.2, + "avg_output_tokens": 9103.8, + "avg_total_tokens": 276128.0, + "avg_turn_count": 21.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/company_in_a_box__goals_restructure.json b/task_results/company_in_a_box__goals_restructure.json index ee87809b..2dcd5144 100644 --- a/task_results/company_in_a_box__goals_restructure.json +++ b/task_results/company_in_a_box__goals_restructure.json @@ -2,6 +2,73 @@ "task": "company_in_a_box__goals_restructure", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 323.6598677635193, + "agent_execution_time": 323.6598677635193, + "token_usage": { + "input_tokens": 765535, + "output_tokens": 2642, + "total_tokens": 768177, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-2", + "success": true, + "execution_time": 2635.1523876190186, + "agent_execution_time": 2635.1523876190186, + "token_usage": { + "input_tokens": 300996, + "output_tokens": 58621, + "total_tokens": 359617, + "reasoning_tokens": 55900 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": true, + "execution_time": 263.1151399612427, + "agent_execution_time": 263.1151399612427, + "token_usage": { + "input_tokens": 589729, + "output_tokens": 2378, + "total_tokens": 592107, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-4", + "success": false, + "execution_time": 423.7647581100464, + "agent_execution_time": 423.7647581100464, + "token_usage": { + "input_tokens": 84801, + "output_tokens": 11370, + "total_tokens": 96171, + "reasoning_tokens": 10988 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 911.42, + "avg_input_tokens": 435265.2, + "avg_output_tokens": 18752.8, + "avg_total_tokens": 454018.0, + "avg_turn_count": 20.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 286.6263744831085, - "agent_execution_time": 286.6263744831085, - "token_usage": { - "input_tokens": 374244, - "output_tokens": 3049, - "total_tokens": 377293, - "reasoning_tokens": null - }, - "turn_count": 19 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 286.63, - "avg_input_tokens": 374244.0, - "avg_output_tokens": 3049.0, - "avg_total_tokens": 377293.0, - "avg_turn_count": 19.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 7.180706739425659, + "agent_execution_time": 7.180706739425659, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-2", + "success": false, + "execution_time": 106.1369080543518, + "agent_execution_time": 106.1369080543518, + "token_usage": { + "input_tokens": 142522, + "output_tokens": 4559, + "total_tokens": 147081, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": true, + "execution_time": 109.21507000923157, + "agent_execution_time": 109.21507000923157, + "token_usage": { + "input_tokens": 267423, + "output_tokens": 4115, + "total_tokens": 271538, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": false, + "execution_time": 138.61377835273743, + "agent_execution_time": 138.61377835273743, + "token_usage": { + "input_tokens": 939219, + "output_tokens": 4143, + "total_tokens": 943362, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 90.29, + "avg_input_tokens": 337291.0, + "avg_output_tokens": 3204.2, + "avg_total_tokens": 340495.2, + "avg_turn_count": 8.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 324.9054024219513, + "agent_execution_time": 324.9054024219513, + "token_usage": { + "input_tokens": 2220243, + "output_tokens": 3266, + "total_tokens": 2223509, + "reasoning_tokens": 0 + }, + "turn_count": 48 + }, + { + "run": "run-2", + "success": true, + "execution_time": 110.3293559551239, + "agent_execution_time": 110.3293559551239, + "token_usage": { + "input_tokens": 442067, + "output_tokens": 2007, + "total_tokens": 444074, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-3", + "success": true, + "execution_time": 229.26349258422852, + "agent_execution_time": 229.26349258422852, + "token_usage": { + "input_tokens": 634486, + "output_tokens": 5096, + "total_tokens": 639582, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-4", + "success": true, + "execution_time": 328.98514771461487, + "agent_execution_time": 328.98514771461487, + "token_usage": { + "input_tokens": 508801, + "output_tokens": 3659, + "total_tokens": 512460, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 248.37, + "avg_input_tokens": 951399.2, + "avg_output_tokens": 3507.0, + "avg_total_tokens": 954906.2, + "avg_turn_count": 31.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 255.54564332962036, + "agent_execution_time": 255.54564332962036, + "token_usage": { + "input_tokens": 526049, + "output_tokens": 2235, + "total_tokens": 528284, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": true, + "execution_time": 250.75043869018555, + "agent_execution_time": 250.75043869018555, + "token_usage": { + "input_tokens": 405411, + "output_tokens": 2450, + "total_tokens": 407861, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": true, + "execution_time": 387.35196185112, + "agent_execution_time": 387.35196185112, + "token_usage": { + "input_tokens": 2260266, + "output_tokens": 2528, + "total_tokens": 2262794, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": true, + "execution_time": 324.52826261520386, + "agent_execution_time": 324.52826261520386, + "token_usage": { + "input_tokens": 1252146, + "output_tokens": 2975, + "total_tokens": 1255121, + "reasoning_tokens": 0 + }, + "turn_count": 33 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 304.54, + "avg_input_tokens": 1110968.0, + "avg_output_tokens": 2547.0, + "avg_total_tokens": 1113515.0, + "avg_turn_count": 26.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 89.88013482093811, + "agent_execution_time": 89.88013482093811, + "token_usage": { + "input_tokens": 265275, + "output_tokens": 7482, + "total_tokens": 272757, + "reasoning_tokens": 6539 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 73.29062843322754, + "agent_execution_time": 73.29062843322754, + "token_usage": { + "input_tokens": 184815, + "output_tokens": 7497, + "total_tokens": 192312, + "reasoning_tokens": 6548 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 81.7825391292572, + "agent_execution_time": 81.7825391292572, + "token_usage": { + "input_tokens": 233964, + "output_tokens": 6802, + "total_tokens": 240766, + "reasoning_tokens": 5624 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 71.3629059791565, + "agent_execution_time": 71.3629059791565, + "token_usage": { + "input_tokens": 142508, + "output_tokens": 8480, + "total_tokens": 150988, + "reasoning_tokens": 7512 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 79.08, + "avg_input_tokens": 206640.5, + "avg_output_tokens": 7565.2, + "avg_total_tokens": 214205.8, + "avg_turn_count": 12.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/company_in_a_box__quarterly_review_dashboard.json b/task_results/company_in_a_box__quarterly_review_dashboard.json index 5b14c95a..cd259458 100644 --- a/task_results/company_in_a_box__quarterly_review_dashboard.json +++ b/task_results/company_in_a_box__quarterly_review_dashboard.json @@ -2,6 +2,73 @@ "task": "company_in_a_box__quarterly_review_dashboard", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 491.57786679267883, + "agent_execution_time": 491.57786679267883, + "token_usage": { + "input_tokens": 1646630, + "output_tokens": 4840, + "total_tokens": 1651470, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + { + "run": "run-2", + "success": false, + "execution_time": 3055.5822019577026, + "agent_execution_time": 3055.5822019577026, + "token_usage": { + "input_tokens": 427190, + "output_tokens": 65443, + "total_tokens": 492633, + "reasoning_tokens": 61266 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 135.1262640953064, + "agent_execution_time": 135.1262640953064, + "token_usage": { + "input_tokens": 298666, + "output_tokens": 1260, + "total_tokens": 299926, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 2390.3604912757874, + "agent_execution_time": 2390.3604912757874, + "token_usage": { + "input_tokens": 498080, + "output_tokens": 63478, + "total_tokens": 561558, + "reasoning_tokens": 55685 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1518.16, + "avg_input_tokens": 717641.5, + "avg_output_tokens": 33755.2, + "avg_total_tokens": 751396.8, + "avg_turn_count": 23.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 497.55545115470886, - "agent_execution_time": 497.55545115470886, - "token_usage": { - "input_tokens": 824570, - "output_tokens": 6513, - "total_tokens": 831083, - "reasoning_tokens": null - }, - "turn_count": 25 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 497.56, - "avg_input_tokens": 824570.0, - "avg_output_tokens": 6513.0, - "avg_total_tokens": 831083.0, - "avg_turn_count": 25.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 7.286128044128418, + "agent_execution_time": 7.286128044128418, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-2", + "success": false, + "execution_time": 303.9621379375458, + "agent_execution_time": 303.9621379375458, + "token_usage": { + "input_tokens": 1646641, + "output_tokens": 10655, + "total_tokens": 1657296, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + { + "run": "run-3", + "success": false, + "execution_time": 503.8980305194855, + "agent_execution_time": 503.8980305194855, + "token_usage": { + "input_tokens": 2113595, + "output_tokens": 11382, + "total_tokens": 2124977, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + { + "run": "run-4", + "success": false, + "execution_time": 403.12762999534607, + "agent_execution_time": 403.12762999534607, + "token_usage": { + "input_tokens": 3463729, + "output_tokens": 10265, + "total_tokens": 3473994, + "reasoning_tokens": 0 + }, + "turn_count": 40 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 304.57, + "avg_input_tokens": 1805991.2, + "avg_output_tokens": 8075.5, + "avg_total_tokens": 1814066.8, + "avg_turn_count": 29.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 806.2172708511353, + "agent_execution_time": 806.2172708511353, + "token_usage": { + "input_tokens": 4155605, + "output_tokens": 11648, + "total_tokens": 4167253, + "reasoning_tokens": 0 + }, + "turn_count": 48 + }, + { + "run": "run-2", + "success": false, + "execution_time": 275.26393938064575, + "agent_execution_time": 275.26393938064575, + "token_usage": { + "input_tokens": 1441421, + "output_tokens": 5647, + "total_tokens": 1447068, + "reasoning_tokens": 0 + }, + "turn_count": 43 + }, + { + "run": "run-3", + "success": false, + "execution_time": 301.31517577171326, + "agent_execution_time": 301.31517577171326, + "token_usage": { + "input_tokens": 1073899, + "output_tokens": 6001, + "total_tokens": 1079900, + "reasoning_tokens": 0 + }, + "turn_count": 41 + }, + { + "run": "run-4", + "success": false, + "execution_time": 746.6911749839783, + "agent_execution_time": 746.6911749839783, + "token_usage": { + "input_tokens": 1945983, + "output_tokens": 8320, + "total_tokens": 1954303, + "reasoning_tokens": 0 + }, + "turn_count": 53 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 532.37, + "avg_input_tokens": 2154227.0, + "avg_output_tokens": 7904.0, + "avg_total_tokens": 2162131.0, + "avg_turn_count": 46.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 444.87319684028625, + "agent_execution_time": 444.87319684028625, + "token_usage": { + "input_tokens": 1295955, + "output_tokens": 4674, + "total_tokens": 1300629, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + { + "run": "run-2", + "success": false, + "execution_time": 9.161523342132568, + "agent_execution_time": 9.161523342132568, + "token_usage": { + "input_tokens": 6858, + "output_tokens": 106, + "total_tokens": 6964, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": false, + "execution_time": 720.7286939620972, + "agent_execution_time": 720.7286939620972, + "token_usage": { + "input_tokens": 2342881, + "output_tokens": 8105, + "total_tokens": 2350986, + "reasoning_tokens": 0 + }, + "turn_count": 52 + }, + { + "run": "run-4", + "success": false, + "execution_time": 377.1310796737671, + "agent_execution_time": 377.1310796737671, + "token_usage": { + "input_tokens": 1046479, + "output_tokens": 4124, + "total_tokens": 1050603, + "reasoning_tokens": 0 + }, + "turn_count": 31 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 387.97, + "avg_input_tokens": 1173043.2, + "avg_output_tokens": 4252.2, + "avg_total_tokens": 1177295.5, + "avg_turn_count": 30.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 104.81969857215881, + "agent_execution_time": 104.81969857215881, + "token_usage": { + "input_tokens": 312438, + "output_tokens": 6789, + "total_tokens": 319227, + "reasoning_tokens": 5274 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 592.4784758090973, + "agent_execution_time": 592.4784758090973, + "token_usage": { + "input_tokens": 64955, + "output_tokens": 117513, + "total_tokens": 182468, + "reasoning_tokens": 2823 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 86.93055653572083, + "agent_execution_time": 86.93055653572083, + "token_usage": { + "input_tokens": 263949, + "output_tokens": 6796, + "total_tokens": 270745, + "reasoning_tokens": 4090 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 173.68438863754272, + "agent_execution_time": 173.68438863754272, + "token_usage": { + "input_tokens": 960578, + "output_tokens": 15556, + "total_tokens": 976134, + "reasoning_tokens": 8531 + }, + "turn_count": 41 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 239.48, + "avg_input_tokens": 400480.0, + "avg_output_tokens": 36663.5, + "avg_total_tokens": 437143.5, + "avg_turn_count": 20.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/computer_science_student_dashboard__code_snippets_go.json b/task_results/computer_science_student_dashboard__code_snippets_go.json index 232746a9..1625683a 100644 --- a/task_results/computer_science_student_dashboard__code_snippets_go.json +++ b/task_results/computer_science_student_dashboard__code_snippets_go.json @@ -2,6 +2,73 @@ "task": "computer_science_student_dashboard__code_snippets_go", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 1187.0391972064972, + "agent_execution_time": 1187.0391972064972, + "token_usage": { + "input_tokens": 2858198, + "output_tokens": 15258, + "total_tokens": 2873456, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-2", + "success": true, + "execution_time": 1056.1947813034058, + "agent_execution_time": 1056.1947813034058, + "token_usage": { + "input_tokens": 150854, + "output_tokens": 24738, + "total_tokens": 175592, + "reasoning_tokens": 23142 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 204.9584858417511, + "agent_execution_time": 204.9584858417511, + "token_usage": { + "input_tokens": 275104, + "output_tokens": 2642, + "total_tokens": 277746, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 817.3883492946625, + "agent_execution_time": 817.3883492946625, + "token_usage": { + "input_tokens": 90491, + "output_tokens": 22676, + "total_tokens": 113167, + "reasoning_tokens": 21364 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 816.4, + "avg_input_tokens": 843661.8, + "avg_output_tokens": 16328.5, + "avg_total_tokens": 859990.2, + "avg_turn_count": 33.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 244.7666482925415, - "agent_execution_time": 244.7666482925415, - "token_usage": { - "input_tokens": 169301, - "output_tokens": 3924, - "total_tokens": 173225, - "reasoning_tokens": null - }, - "turn_count": 11 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 244.77, - "avg_input_tokens": 169301.0, - "avg_output_tokens": 3924.0, - "avg_total_tokens": 173225.0, - "avg_turn_count": 11.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 189.04399418830872, + "agent_execution_time": 189.04399418830872, + "token_usage": { + "input_tokens": 166723, + "output_tokens": 2885, + "total_tokens": 169608, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": true, + "execution_time": 75.76531982421875, + "agent_execution_time": 75.76531982421875, + "token_usage": { + "input_tokens": 142324, + "output_tokens": 2939, + "total_tokens": 145263, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": true, + "execution_time": 76.72847747802734, + "agent_execution_time": 76.72847747802734, + "token_usage": { + "input_tokens": 140643, + "output_tokens": 2756, + "total_tokens": 143399, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 114.53728413581848, + "agent_execution_time": 114.53728413581848, + "token_usage": { + "input_tokens": 1042557, + "output_tokens": 2293, + "total_tokens": 1044850, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 114.02, + "avg_input_tokens": 373061.8, + "avg_output_tokens": 2718.2, + "avg_total_tokens": 375780.0, + "avg_turn_count": 10.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 1152.8275332450867, + "agent_execution_time": 1152.8275332450867, + "token_usage": { + "input_tokens": 3374753, + "output_tokens": 28840, + "total_tokens": 3403593, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-2", + "success": false, + "execution_time": 569.8524651527405, + "agent_execution_time": 569.8524651527405, + "token_usage": { + "input_tokens": 2050260, + "output_tokens": 15308, + "total_tokens": 2065568, + "reasoning_tokens": 0 + }, + "turn_count": 56 + }, + { + "run": "run-3", + "success": true, + "execution_time": 169.74611163139343, + "agent_execution_time": 169.74611163139343, + "token_usage": { + "input_tokens": 259855, + "output_tokens": 3940, + "total_tokens": 263795, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 3048.3186750411987, + "agent_execution_time": 3048.3186750411987, + "token_usage": { + "input_tokens": 605329, + "output_tokens": 10508, + "total_tokens": 615837, + "reasoning_tokens": 0 + }, + "turn_count": 33 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 1235.19, + "avg_input_tokens": 1572549.2, + "avg_output_tokens": 14649.0, + "avg_total_tokens": 1587198.2, + "avg_turn_count": 52.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 207.80923223495483, + "agent_execution_time": 207.80923223495483, + "token_usage": { + "input_tokens": 321688, + "output_tokens": 2068, + "total_tokens": 323756, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 7.575263500213623, + "agent_execution_time": 7.575263500213623, + "token_usage": { + "input_tokens": 6712, + "output_tokens": 99, + "total_tokens": 6811, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": true, + "execution_time": 517.6202044487, + "agent_execution_time": 517.6202044487, + "token_usage": { + "input_tokens": 867599, + "output_tokens": 6688, + "total_tokens": 874287, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + { + "run": "run-4", + "success": false, + "execution_time": 205.75380516052246, + "agent_execution_time": 205.75380516052246, + "token_usage": { + "input_tokens": 310797, + "output_tokens": 2372, + "total_tokens": 313169, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 234.69, + "avg_input_tokens": 376699.0, + "avg_output_tokens": 2806.8, + "avg_total_tokens": 379505.8, + "avg_turn_count": 18.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 152.2839071750641, + "agent_execution_time": 152.2839071750641, + "token_usage": { + "input_tokens": 431250, + "output_tokens": 6865, + "total_tokens": 438115, + "reasoning_tokens": 5144 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": false, + "execution_time": 90.05319833755493, + "agent_execution_time": 90.05319833755493, + "token_usage": { + "input_tokens": 338050, + "output_tokens": 5862, + "total_tokens": 343912, + "reasoning_tokens": 4509 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": false, + "execution_time": 72.62897086143494, + "agent_execution_time": 72.62897086143494, + "token_usage": { + "input_tokens": 442974, + "output_tokens": 5445, + "total_tokens": 448419, + "reasoning_tokens": 3707 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 117.43509840965271, + "agent_execution_time": 117.43509840965271, + "token_usage": { + "input_tokens": 506019, + "output_tokens": 9186, + "total_tokens": 515205, + "reasoning_tokens": 7181 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 108.1, + "avg_input_tokens": 429573.2, + "avg_output_tokens": 6839.5, + "avg_total_tokens": 436412.8, + "avg_turn_count": 23.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/computer_science_student_dashboard__courses_internships_relation.json b/task_results/computer_science_student_dashboard__courses_internships_relation.json index 12ada5c0..1ca0b61a 100644 --- a/task_results/computer_science_student_dashboard__courses_internships_relation.json +++ b/task_results/computer_science_student_dashboard__courses_internships_relation.json @@ -2,6 +2,73 @@ "task": "computer_science_student_dashboard__courses_internships_relation", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 277.7262635231018, + "agent_execution_time": 277.7262635231018, + "token_usage": { + "input_tokens": 618412, + "output_tokens": 2972, + "total_tokens": 621384, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": false, + "execution_time": 2886.3092391490936, + "agent_execution_time": 2886.3092391490936, + "token_usage": { + "input_tokens": 237041, + "output_tokens": 61844, + "total_tokens": 298885, + "reasoning_tokens": 57715 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 118.15512609481812, + "agent_execution_time": 118.15512609481812, + "token_usage": { + "input_tokens": 138966, + "output_tokens": 1347, + "total_tokens": 140313, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1408.530945777893, + "agent_execution_time": 1408.530945777893, + "token_usage": { + "input_tokens": 157236, + "output_tokens": 38864, + "total_tokens": 196100, + "reasoning_tokens": 36770 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1172.68, + "avg_input_tokens": 287913.8, + "avg_output_tokens": 26256.8, + "avg_total_tokens": 314170.5, + "avg_turn_count": 15.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 363.53436517715454, - "agent_execution_time": 363.53436517715454, - "token_usage": { - "input_tokens": 420966, - "output_tokens": 4170, - "total_tokens": 425136, - "reasoning_tokens": null - }, - "turn_count": 21 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 363.53, - "avg_input_tokens": 420966.0, - "avg_output_tokens": 4170.0, - "avg_total_tokens": 425136.0, - "avg_turn_count": 21.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 132.7680013179779, + "agent_execution_time": 132.7680013179779, + "token_usage": { + "input_tokens": 244992, + "output_tokens": 3650, + "total_tokens": 248642, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 179.38031792640686, + "agent_execution_time": 179.38031792640686, + "token_usage": { + "input_tokens": 446478, + "output_tokens": 6153, + "total_tokens": 452631, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 402.04554772377014, + "agent_execution_time": 402.04554772377014, + "token_usage": { + "input_tokens": 1136128, + "output_tokens": 7014, + "total_tokens": 1143142, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 401.2056860923767, + "agent_execution_time": 401.2056860923767, + "token_usage": { + "input_tokens": 1845772, + "output_tokens": 11382, + "total_tokens": 1857154, + "reasoning_tokens": 0 + }, + "turn_count": 30 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 278.85, + "avg_input_tokens": 918342.5, + "avg_output_tokens": 7049.8, + "avg_total_tokens": 925392.2, + "avg_turn_count": 19.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 228.92362999916077, + "agent_execution_time": 228.92362999916077, + "token_usage": { + "input_tokens": 339961, + "output_tokens": 4803, + "total_tokens": 344764, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 136.8230242729187, + "agent_execution_time": 136.8230242729187, + "token_usage": { + "input_tokens": 465047, + "output_tokens": 2431, + "total_tokens": 467478, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-3", + "success": false, + "execution_time": 184.4326102733612, + "agent_execution_time": 184.4326102733612, + "token_usage": { + "input_tokens": 373668, + "output_tokens": 5746, + "total_tokens": 379414, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 232.4560890197754, + "agent_execution_time": 232.4560890197754, + "token_usage": { + "input_tokens": 381947, + "output_tokens": 4908, + "total_tokens": 386855, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 195.66, + "avg_input_tokens": 390155.8, + "avg_output_tokens": 4472.0, + "avg_total_tokens": 394627.8, + "avg_turn_count": 22.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 319.4499981403351, + "agent_execution_time": 319.4499981403351, + "token_usage": { + "input_tokens": 633921, + "output_tokens": 3584, + "total_tokens": 637505, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-2", + "success": false, + "execution_time": 7.643780946731567, + "agent_execution_time": 7.643780946731567, + "token_usage": { + "input_tokens": 6886, + "output_tokens": 102, + "total_tokens": 6988, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": false, + "execution_time": 328.9060800075531, + "agent_execution_time": 328.9060800075531, + "token_usage": { + "input_tokens": 447623, + "output_tokens": 3449, + "total_tokens": 451072, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": false, + "execution_time": 32.541977405548096, + "agent_execution_time": 32.541977405548096, + "token_usage": { + "input_tokens": 26376, + "output_tokens": 307, + "total_tokens": 26683, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 172.14, + "avg_input_tokens": 278701.5, + "avg_output_tokens": 1860.5, + "avg_total_tokens": 280562.0, + "avg_turn_count": 15.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 137.83059763908386, + "agent_execution_time": 137.83059763908386, + "token_usage": { + "input_tokens": 746141, + "output_tokens": 10248, + "total_tokens": 756389, + "reasoning_tokens": 7360 + }, + "turn_count": 32 + }, + { + "run": "run-2", + "success": false, + "execution_time": 108.66051697731018, + "agent_execution_time": 108.66051697731018, + "token_usage": { + "input_tokens": 496192, + "output_tokens": 8675, + "total_tokens": 504867, + "reasoning_tokens": 6730 + }, + "turn_count": 28 + }, + { + "run": "run-3", + "success": false, + "execution_time": 118.29310750961304, + "agent_execution_time": 118.29310750961304, + "token_usage": { + "input_tokens": 338355, + "output_tokens": 8281, + "total_tokens": 346636, + "reasoning_tokens": 6435 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 147.53497695922852, + "agent_execution_time": 147.53497695922852, + "token_usage": { + "input_tokens": 639194, + "output_tokens": 11619, + "total_tokens": 650813, + "reasoning_tokens": 8596 + }, + "turn_count": 32 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 128.08, + "avg_input_tokens": 554970.5, + "avg_output_tokens": 9705.8, + "avg_total_tokens": 564676.2, + "avg_turn_count": 29.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/computer_science_student_dashboard__study_session_tracker.json b/task_results/computer_science_student_dashboard__study_session_tracker.json index 96dac8a7..36a0ee48 100644 --- a/task_results/computer_science_student_dashboard__study_session_tracker.json +++ b/task_results/computer_science_student_dashboard__study_session_tracker.json @@ -2,6 +2,73 @@ "task": "computer_science_student_dashboard__study_session_tracker", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 101.82571601867676, + "agent_execution_time": 101.82571601867676, + "token_usage": { + "input_tokens": 130432, + "output_tokens": 1098, + "total_tokens": 131530, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1246.9643194675446, + "agent_execution_time": 1246.9643194675446, + "token_usage": { + "input_tokens": 152717, + "output_tokens": 26813, + "total_tokens": 179530, + "reasoning_tokens": 25489 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 120.59121704101562, + "agent_execution_time": 120.59121704101562, + "token_usage": { + "input_tokens": 136996, + "output_tokens": 1293, + "total_tokens": 138289, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 650.3801589012146, + "agent_execution_time": 650.3801589012146, + "token_usage": { + "input_tokens": 116499, + "output_tokens": 16871, + "total_tokens": 133370, + "reasoning_tokens": 15726 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 529.94, + "avg_input_tokens": 134161.0, + "avg_output_tokens": 11518.8, + "avg_total_tokens": 145679.8, + "avg_turn_count": 9.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 275.5879247188568, - "agent_execution_time": 275.5879247188568, - "token_usage": { - "input_tokens": 330908, - "output_tokens": 3221, - "total_tokens": 334129, - "reasoning_tokens": null - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 275.59, - "avg_input_tokens": 330908.0, - "avg_output_tokens": 3221.0, - "avg_total_tokens": 334129.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 157.88507723808289, + "agent_execution_time": 157.88507723808289, + "token_usage": { + "input_tokens": 241738, + "output_tokens": 4129, + "total_tokens": 245867, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 64.28976011276245, + "agent_execution_time": 64.28976011276245, + "token_usage": { + "input_tokens": 112607, + "output_tokens": 2244, + "total_tokens": 114851, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 230.87277221679688, + "agent_execution_time": 230.87277221679688, + "token_usage": { + "input_tokens": 644963, + "output_tokens": 7011, + "total_tokens": 651974, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 45.71245288848877, + "agent_execution_time": 45.71245288848877, + "token_usage": { + "input_tokens": 82118, + "output_tokens": 1779, + "total_tokens": 83897, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 124.69, + "avg_input_tokens": 270356.5, + "avg_output_tokens": 3790.8, + "avg_total_tokens": 274147.2, + "avg_turn_count": 11.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 83.70585131645203, + "agent_execution_time": 83.70585131645203, + "token_usage": { + "input_tokens": 108017, + "output_tokens": 2458, + "total_tokens": 110475, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 85.42530083656311, + "agent_execution_time": 85.42530083656311, + "token_usage": { + "input_tokens": 272294, + "output_tokens": 1527, + "total_tokens": 273821, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 111.19457936286926, + "agent_execution_time": 111.19457936286926, + "token_usage": { + "input_tokens": 107291, + "output_tokens": 2553, + "total_tokens": 109844, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 95.33345341682434, + "agent_execution_time": 95.33345341682434, + "token_usage": { + "input_tokens": 113697, + "output_tokens": 2453, + "total_tokens": 116150, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 93.91, + "avg_input_tokens": 150324.8, + "avg_output_tokens": 2247.8, + "avg_total_tokens": 152572.5, + "avg_turn_count": 10.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 170.7622299194336, + "agent_execution_time": 170.7622299194336, + "token_usage": { + "input_tokens": 125193, + "output_tokens": 1188, + "total_tokens": 126381, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 215.09980010986328, + "agent_execution_time": 215.09980010986328, + "token_usage": { + "input_tokens": 431310, + "output_tokens": 1208, + "total_tokens": 432518, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 108.98834300041199, + "agent_execution_time": 108.98834300041199, + "token_usage": { + "input_tokens": 124620, + "output_tokens": 1147, + "total_tokens": 125767, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 112.19266366958618, + "agent_execution_time": 112.19266366958618, + "token_usage": { + "input_tokens": 143512, + "output_tokens": 1269, + "total_tokens": 144781, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 151.76, + "avg_input_tokens": 206158.8, + "avg_output_tokens": 1203.0, + "avg_total_tokens": 207361.8, + "avg_turn_count": 11.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 734.4401268959045, + "agent_execution_time": 734.4401268959045, + "token_usage": { + "input_tokens": 559753, + "output_tokens": 8346, + "total_tokens": 568099, + "reasoning_tokens": 5860 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 596.5692698955536, + "agent_execution_time": 596.5692698955536, + "token_usage": { + "input_tokens": 104787, + "output_tokens": 114695, + "total_tokens": 219482, + "reasoning_tokens": 4280 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 106.04337930679321, + "agent_execution_time": 106.04337930679321, + "token_usage": { + "input_tokens": 148630, + "output_tokens": 15641, + "total_tokens": 164271, + "reasoning_tokens": 6369 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 70.91306114196777, + "agent_execution_time": 70.91306114196777, + "token_usage": { + "input_tokens": 258821, + "output_tokens": 6318, + "total_tokens": 265139, + "reasoning_tokens": 4860 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 376.99, + "avg_input_tokens": 267997.8, + "avg_output_tokens": 36250.0, + "avg_total_tokens": 304247.8, + "avg_turn_count": 15.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/desktop__music_report.json b/task_results/desktop__music_report.json index 56abaef2..390c0257 100644 --- a/task_results/desktop__music_report.json +++ b/task_results/desktop__music_report.json @@ -2,6 +2,73 @@ "task": "desktop__music_report", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 247.4196126461029, + "agent_execution_time": 247.4196126461029, + "token_usage": { + "input_tokens": 36556, + "output_tokens": 5128, + "total_tokens": 41684, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": true, + "execution_time": 173.90516328811646, + "agent_execution_time": 173.90516328811646, + "token_usage": { + "input_tokens": 44065, + "output_tokens": 3045, + "total_tokens": 47110, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": true, + "execution_time": 614.7150180339813, + "agent_execution_time": 614.7150180339813, + "token_usage": { + "input_tokens": 36101, + "output_tokens": 15219, + "total_tokens": 51320, + "reasoning_tokens": 14477 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 87.21180248260498, + "agent_execution_time": 87.21180248260498, + "token_usage": { + "input_tokens": 9918, + "output_tokens": 1871, + "total_tokens": 11789, + "reasoning_tokens": 1772 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 280.81, + "avg_input_tokens": 31660.0, + "avg_output_tokens": 6315.8, + "avg_total_tokens": 37975.8, + "avg_turn_count": 8.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 98.89563918113708, - "agent_execution_time": 98.89563918113708, - "token_usage": { - "input_tokens": 25612, - "output_tokens": 1527, - "total_tokens": 27139, - "reasoning_tokens": null - }, - "turn_count": 8 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 98.9, - "avg_input_tokens": 25612.0, - "avg_output_tokens": 1527.0, - "avg_total_tokens": 27139.0, - "avg_turn_count": 8.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 7.522918462753296, + "agent_execution_time": 7.522918462753296, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-2", + "success": true, + "execution_time": 49.86474108695984, + "agent_execution_time": 49.86474108695984, + "token_usage": { + "input_tokens": 37772, + "output_tokens": 3309, + "total_tokens": 41081, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": true, + "execution_time": 48.36807107925415, + "agent_execution_time": 48.36807107925415, + "token_usage": { + "input_tokens": 37653, + "output_tokens": 3298, + "total_tokens": 40951, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": true, + "execution_time": 48.838834285736084, + "agent_execution_time": 48.838834285736084, + "token_usage": { + "input_tokens": 37709, + "output_tokens": 3270, + "total_tokens": 40979, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 38.65, + "avg_input_tokens": 28283.5, + "avg_output_tokens": 2469.2, + "avg_total_tokens": 30752.8, + "avg_turn_count": 4.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 31.723867654800415, + "agent_execution_time": 31.723867654800415, + "token_usage": { + "input_tokens": 24418, + "output_tokens": 961, + "total_tokens": 25379, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 15.10427474975586, + "agent_execution_time": 15.10427474975586, + "token_usage": { + "input_tokens": 22831, + "output_tokens": 1097, + "total_tokens": 23928, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 31.723867654800415, + "agent_execution_time": 31.723867654800415, + "token_usage": { + "input_tokens": 24418, + "output_tokens": 961, + "total_tokens": 25379, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 15.10427474975586, + "agent_execution_time": 15.10427474975586, + "token_usage": { + "input_tokens": 22831, + "output_tokens": 1097, + "total_tokens": 23928, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 23.41, + "avg_input_tokens": 23624.5, + "avg_output_tokens": 1029.0, + "avg_total_tokens": 24653.5, + "avg_turn_count": 7.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 129.30624628067017, + "agent_execution_time": 129.30624628067017, + "token_usage": { + "input_tokens": 52687, + "output_tokens": 1769, + "total_tokens": 54456, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 118.57634544372559, + "agent_execution_time": 118.57634544372559, + "token_usage": { + "input_tokens": 47130, + "output_tokens": 1709, + "total_tokens": 48839, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 79.30341839790344, + "agent_execution_time": 79.30341839790344, + "token_usage": { + "input_tokens": 36390, + "output_tokens": 1619, + "total_tokens": 38009, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 52.89259696006775, + "agent_execution_time": 52.89259696006775, + "token_usage": { + "input_tokens": 28004, + "output_tokens": 855, + "total_tokens": 28859, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 95.02, + "avg_input_tokens": 41052.8, + "avg_output_tokens": 1488.0, + "avg_total_tokens": 42540.8, + "avg_turn_count": 10.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 42.31699299812317, + "agent_execution_time": 42.31699299812317, + "token_usage": { + "input_tokens": 38096, + "output_tokens": 5484, + "total_tokens": 43580, + "reasoning_tokens": 4929 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 47.07411599159241, + "agent_execution_time": 47.07411599159241, + "token_usage": { + "input_tokens": 37741, + "output_tokens": 5306, + "total_tokens": 43047, + "reasoning_tokens": 4697 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 38.21270203590393, + "agent_execution_time": 38.21270203590393, + "token_usage": { + "input_tokens": 36783, + "output_tokens": 4806, + "total_tokens": 41589, + "reasoning_tokens": 4198 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 41.401585817337036, + "agent_execution_time": 41.401585817337036, + "token_usage": { + "input_tokens": 37032, + "output_tokens": 4835, + "total_tokens": 41867, + "reasoning_tokens": 4226 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 42.25, + "avg_input_tokens": 37413.0, + "avg_output_tokens": 5107.8, + "avg_total_tokens": 42520.8, + "avg_turn_count": 10.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/desktop__project_management.json b/task_results/desktop__project_management.json index 0494ae17..5dc31403 100644 --- a/task_results/desktop__project_management.json +++ b/task_results/desktop__project_management.json @@ -2,6 +2,73 @@ "task": "desktop__project_management", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 340.08776688575745, + "agent_execution_time": 340.08776688575745, + "token_usage": { + "input_tokens": 283339, + "output_tokens": 3971, + "total_tokens": 287310, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + { + "run": "run-2", + "success": false, + "execution_time": 405.20643281936646, + "agent_execution_time": 405.20643281936646, + "token_usage": { + "input_tokens": 413338, + "output_tokens": 4891, + "total_tokens": 418229, + "reasoning_tokens": 0 + }, + "turn_count": 46 + }, + { + "run": "run-3", + "success": false, + "execution_time": 983.6114149093628, + "agent_execution_time": 983.6114149093628, + "token_usage": { + "input_tokens": 65895, + "output_tokens": 22154, + "total_tokens": 88049, + "reasoning_tokens": 19806 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1229.6339173316956, + "agent_execution_time": 1229.6339173316956, + "token_usage": { + "input_tokens": 99460, + "output_tokens": 32049, + "total_tokens": 131509, + "reasoning_tokens": 27719 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 739.63, + "avg_input_tokens": 215508.0, + "avg_output_tokens": 15766.2, + "avg_total_tokens": 231274.2, + "avg_turn_count": 28.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 559.8005208969116, - "agent_execution_time": 559.8005208969116, - "token_usage": { - "input_tokens": 552323, - "output_tokens": 9255, - "total_tokens": 561578, - "reasoning_tokens": null - }, - "turn_count": 51 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 559.8, - "avg_input_tokens": 552323.0, - "avg_output_tokens": 9255.0, - "avg_total_tokens": 561578.0, - "avg_turn_count": 51.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 167.9004487991333, + "agent_execution_time": 167.9004487991333, + "token_usage": { + "input_tokens": 168532, + "output_tokens": 7416, + "total_tokens": 175948, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": true, + "execution_time": 119.9151623249054, + "agent_execution_time": 119.9151623249054, + "token_usage": { + "input_tokens": 168190, + "output_tokens": 7133, + "total_tokens": 175323, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": true, + "execution_time": 113.30031991004944, + "agent_execution_time": 113.30031991004944, + "token_usage": { + "input_tokens": 158346, + "output_tokens": 7538, + "total_tokens": 165884, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": true, + "execution_time": 130.4593415260315, + "agent_execution_time": 130.4593415260315, + "token_usage": { + "input_tokens": 202823, + "output_tokens": 8231, + "total_tokens": 211054, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 132.89, + "avg_input_tokens": 174472.8, + "avg_output_tokens": 7579.5, + "avg_total_tokens": 182052.2, + "avg_turn_count": 15.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 125.68024682998657, + "agent_execution_time": 125.68024682998657, + "token_usage": { + "input_tokens": 442751, + "output_tokens": 7037, + "total_tokens": 449788, + "reasoning_tokens": 0 + }, + "turn_count": 48 + }, + { + "run": "run-2", + "success": false, + "execution_time": 132.5841419696808, + "agent_execution_time": 132.5841419696808, + "token_usage": { + "input_tokens": 603557, + "output_tokens": 7514, + "total_tokens": 611071, + "reasoning_tokens": 0 + }, + "turn_count": 52 + }, + { + "run": "run-3", + "success": false, + "execution_time": 125.68024682998657, + "agent_execution_time": 125.68024682998657, + "token_usage": { + "input_tokens": 442751, + "output_tokens": 7037, + "total_tokens": 449788, + "reasoning_tokens": 0 + }, + "turn_count": 48 + }, + { + "run": "run-4", + "success": false, + "execution_time": 132.5841419696808, + "agent_execution_time": 132.5841419696808, + "token_usage": { + "input_tokens": 603557, + "output_tokens": 7514, + "total_tokens": 611071, + "reasoning_tokens": 0 + }, + "turn_count": 52 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 129.13, + "avg_input_tokens": 523154.0, + "avg_output_tokens": 7275.5, + "avg_total_tokens": 530429.5, + "avg_turn_count": 50.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 391.0343129634857, + "agent_execution_time": 391.0343129634857, + "token_usage": { + "input_tokens": 310079, + "output_tokens": 4737, + "total_tokens": 314816, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + { + "run": "run-2", + "success": false, + "execution_time": 353.55652832984924, + "agent_execution_time": 353.55652832984924, + "token_usage": { + "input_tokens": 284959, + "output_tokens": 4228, + "total_tokens": 289187, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + { + "run": "run-3", + "success": false, + "execution_time": 307.50742197036743, + "agent_execution_time": 307.50742197036743, + "token_usage": { + "input_tokens": 118280, + "output_tokens": 6289, + "total_tokens": 124569, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 205.3735911846161, + "agent_execution_time": 205.3735911846161, + "token_usage": { + "input_tokens": 90657, + "output_tokens": 4949, + "total_tokens": 95606, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 314.37, + "avg_input_tokens": 200993.8, + "avg_output_tokens": 5050.8, + "avg_total_tokens": 206044.5, + "avg_turn_count": 27.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 106.77428698539734, + "agent_execution_time": 106.77428698539734, + "token_usage": { + "input_tokens": 171223, + "output_tokens": 10862, + "total_tokens": 182085, + "reasoning_tokens": 5294 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 65.06278777122498, + "agent_execution_time": 65.06278777122498, + "token_usage": { + "input_tokens": 188413, + "output_tokens": 9143, + "total_tokens": 197556, + "reasoning_tokens": 4336 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": true, + "execution_time": 85.98844456672668, + "agent_execution_time": 85.98844456672668, + "token_usage": { + "input_tokens": 198448, + "output_tokens": 10371, + "total_tokens": 208819, + "reasoning_tokens": 5337 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 75.20499229431152, + "agent_execution_time": 75.20499229431152, + "token_usage": { + "input_tokens": 198162, + "output_tokens": 7347, + "total_tokens": 205509, + "reasoning_tokens": 3147 + }, + "turn_count": 28 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 83.26, + "avg_input_tokens": 189061.5, + "avg_output_tokens": 9430.8, + "avg_total_tokens": 198492.2, + "avg_turn_count": 21.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/desktop__timeline_extraction.json b/task_results/desktop__timeline_extraction.json index 08d9fb21..ad203b97 100644 --- a/task_results/desktop__timeline_extraction.json +++ b/task_results/desktop__timeline_extraction.json @@ -2,6 +2,73 @@ "task": "desktop__timeline_extraction", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 129.97008752822876, + "agent_execution_time": 129.97008752822876, + "token_usage": { + "input_tokens": 77219, + "output_tokens": 2438, + "total_tokens": 79657, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 1060.0679953098297, + "agent_execution_time": 1060.0679953098297, + "token_usage": { + "input_tokens": 126262, + "output_tokens": 28481, + "total_tokens": 154743, + "reasoning_tokens": 24102 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 1507.707370042801, + "agent_execution_time": 1507.707370042801, + "token_usage": { + "input_tokens": 96849, + "output_tokens": 36846, + "total_tokens": 133695, + "reasoning_tokens": 33514 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1462.4173436164856, + "agent_execution_time": 1462.4173436164856, + "token_usage": { + "input_tokens": 80330, + "output_tokens": 40539, + "total_tokens": 120869, + "reasoning_tokens": 37710 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 1040.04, + "avg_input_tokens": 95165.0, + "avg_output_tokens": 27076.0, + "avg_total_tokens": 122241.0, + "avg_turn_count": 7.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 240.9384047985077, - "agent_execution_time": 240.9384047985077, - "token_usage": { - "input_tokens": 188507, - "output_tokens": 3860, - "total_tokens": 192367, - "reasoning_tokens": null - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 240.94, - "avg_input_tokens": 188507.0, - "avg_output_tokens": 3860.0, - "avg_total_tokens": 192367.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 81.54736590385437, + "agent_execution_time": 81.54736590385437, + "token_usage": { + "input_tokens": 119386, + "output_tokens": 2893, + "total_tokens": 122279, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 63.116132497787476, + "agent_execution_time": 63.116132497787476, + "token_usage": { + "input_tokens": 104173, + "output_tokens": 3299, + "total_tokens": 107472, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 71.25612831115723, + "agent_execution_time": 71.25612831115723, + "token_usage": { + "input_tokens": 160317, + "output_tokens": 4073, + "total_tokens": 164390, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 71.5654833316803, + "agent_execution_time": 71.5654833316803, + "token_usage": { + "input_tokens": 140307, + "output_tokens": 4304, + "total_tokens": 144611, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 71.87, + "avg_input_tokens": 131045.8, + "avg_output_tokens": 3642.2, + "avg_total_tokens": 134688.0, + "avg_turn_count": 9.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 57.27190375328064, + "agent_execution_time": 57.27190375328064, + "token_usage": { + "input_tokens": 63059, + "output_tokens": 3130, + "total_tokens": 66189, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 26.837627172470093, + "agent_execution_time": 26.837627172470093, + "token_usage": { + "input_tokens": 58364, + "output_tokens": 2648, + "total_tokens": 61012, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 57.27190375328064, + "agent_execution_time": 57.27190375328064, + "token_usage": { + "input_tokens": 63059, + "output_tokens": 3130, + "total_tokens": 66189, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 26.837627172470093, + "agent_execution_time": 26.837627172470093, + "token_usage": { + "input_tokens": 58364, + "output_tokens": 2648, + "total_tokens": 61012, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 42.05, + "avg_input_tokens": 60711.5, + "avg_output_tokens": 2889.0, + "avg_total_tokens": 63600.5, + "avg_turn_count": 9.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 138.61336493492126, + "agent_execution_time": 138.61336493492126, + "token_usage": { + "input_tokens": 58727, + "output_tokens": 2446, + "total_tokens": 61173, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 161.50582480430603, + "agent_execution_time": 161.50582480430603, + "token_usage": { + "input_tokens": 127078, + "output_tokens": 2803, + "total_tokens": 129881, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 126.46522903442383, + "agent_execution_time": 126.46522903442383, + "token_usage": { + "input_tokens": 100187, + "output_tokens": 2126, + "total_tokens": 102313, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 96.26100182533264, + "agent_execution_time": 96.26100182533264, + "token_usage": { + "input_tokens": 56394, + "output_tokens": 2037, + "total_tokens": 58431, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 130.71, + "avg_input_tokens": 85596.5, + "avg_output_tokens": 2353.0, + "avg_total_tokens": 87949.5, + "avg_turn_count": 7.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 49.41848683357239, + "agent_execution_time": 49.41848683357239, + "token_usage": { + "input_tokens": 46409, + "output_tokens": 6445, + "total_tokens": 52854, + "reasoning_tokens": 4977 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 40.88468360900879, + "agent_execution_time": 40.88468360900879, + "token_usage": { + "input_tokens": 64128, + "output_tokens": 7290, + "total_tokens": 71418, + "reasoning_tokens": 5558 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": true, + "execution_time": 50.150161266326904, + "agent_execution_time": 50.150161266326904, + "token_usage": { + "input_tokens": 63016, + "output_tokens": 8631, + "total_tokens": 71647, + "reasoning_tokens": 6849 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 69.8250961303711, + "agent_execution_time": 69.8250961303711, + "token_usage": { + "input_tokens": 355044, + "output_tokens": 6844, + "total_tokens": 361888, + "reasoning_tokens": 4666 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 52.57, + "avg_input_tokens": 132149.2, + "avg_output_tokens": 7302.5, + "avg_total_tokens": 139451.8, + "avg_turn_count": 11.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/desktop_template__budget_computation.json b/task_results/desktop_template__budget_computation.json index 22441600..76669d5e 100644 --- a/task_results/desktop_template__budget_computation.json +++ b/task_results/desktop_template__budget_computation.json @@ -2,6 +2,73 @@ "task": "desktop_template__budget_computation", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 113.12025308609009, + "agent_execution_time": 113.12025308609009, + "token_usage": { + "input_tokens": 49961, + "output_tokens": 1778, + "total_tokens": 51739, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 332.6777753829956, + "agent_execution_time": 332.6777753829956, + "token_usage": { + "input_tokens": 17654, + "output_tokens": 9258, + "total_tokens": 26912, + "reasoning_tokens": 8407 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 621.1506094932556, + "agent_execution_time": 621.1506094932556, + "token_usage": { + "input_tokens": 16739, + "output_tokens": 14011, + "total_tokens": 30750, + "reasoning_tokens": 13644 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 787.7252607345581, + "agent_execution_time": 787.7252607345581, + "token_usage": { + "input_tokens": 25612, + "output_tokens": 21529, + "total_tokens": 47141, + "reasoning_tokens": 20308 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 463.67, + "avg_input_tokens": 27491.5, + "avg_output_tokens": 11644.0, + "avg_total_tokens": 39135.5, + "avg_turn_count": 5.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 125.44478368759155, - "agent_execution_time": 125.44478368759155, - "token_usage": { - "input_tokens": 35389, - "output_tokens": 2139, - "total_tokens": 37528, - "reasoning_tokens": null - }, - "turn_count": 7 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 125.44, - "avg_input_tokens": 35389.0, - "avg_output_tokens": 2139.0, - "avg_total_tokens": 37528.0, - "avg_turn_count": 7.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 90.00802803039551, + "agent_execution_time": 90.00802803039551, + "token_usage": { + "input_tokens": 47006, + "output_tokens": 4058, + "total_tokens": 51064, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 57.92476272583008, + "agent_execution_time": 57.92476272583008, + "token_usage": { + "input_tokens": 59880, + "output_tokens": 2560, + "total_tokens": 62440, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 60.530882358551025, + "agent_execution_time": 60.530882358551025, + "token_usage": { + "input_tokens": 68601, + "output_tokens": 2830, + "total_tokens": 71431, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 53.02128624916077, + "agent_execution_time": 53.02128624916077, + "token_usage": { + "input_tokens": 59940, + "output_tokens": 2356, + "total_tokens": 62296, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 65.37, + "avg_input_tokens": 58856.8, + "avg_output_tokens": 2951.0, + "avg_total_tokens": 61807.8, + "avg_turn_count": 9.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 46.019153356552124, + "agent_execution_time": 46.019153356552124, + "token_usage": { + "input_tokens": 58174, + "output_tokens": 3070, + "total_tokens": 61244, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 16.283254384994507, + "agent_execution_time": 16.283254384994507, + "token_usage": { + "input_tokens": 24004, + "output_tokens": 1281, + "total_tokens": 25285, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 46.019153356552124, + "agent_execution_time": 46.019153356552124, + "token_usage": { + "input_tokens": 58174, + "output_tokens": 3070, + "total_tokens": 61244, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 16.283254384994507, + "agent_execution_time": 16.283254384994507, + "token_usage": { + "input_tokens": 24004, + "output_tokens": 1281, + "total_tokens": 25285, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 31.15, + "avg_input_tokens": 41089.0, + "avg_output_tokens": 2175.5, + "avg_total_tokens": 43264.5, + "avg_turn_count": 9.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 132.58616971969604, + "agent_execution_time": 132.58616971969604, + "token_usage": { + "input_tokens": 66967, + "output_tokens": 1777, + "total_tokens": 68744, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 122.725013256073, + "agent_execution_time": 122.725013256073, + "token_usage": { + "input_tokens": 51556, + "output_tokens": 1934, + "total_tokens": 53490, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 82.5663194656372, + "agent_execution_time": 82.5663194656372, + "token_usage": { + "input_tokens": 44275, + "output_tokens": 1289, + "total_tokens": 45564, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 141.5936632156372, + "agent_execution_time": 141.5936632156372, + "token_usage": { + "input_tokens": 82535, + "output_tokens": 2017, + "total_tokens": 84552, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 119.87, + "avg_input_tokens": 61333.2, + "avg_output_tokens": 1754.2, + "avg_total_tokens": 63087.5, + "avg_turn_count": 11.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 33.91645622253418, + "agent_execution_time": 33.91645622253418, + "token_usage": { + "input_tokens": 94530, + "output_tokens": 2658, + "total_tokens": 97188, + "reasoning_tokens": 1589 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 31.894129037857056, + "agent_execution_time": 31.894129037857056, + "token_usage": { + "input_tokens": 55870, + "output_tokens": 4269, + "total_tokens": 60139, + "reasoning_tokens": 3147 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 43.58357357978821, + "agent_execution_time": 43.58357357978821, + "token_usage": { + "input_tokens": 50029, + "output_tokens": 4146, + "total_tokens": 54175, + "reasoning_tokens": 2983 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 45.23974370956421, + "agent_execution_time": 45.23974370956421, + "token_usage": { + "input_tokens": 95833, + "output_tokens": 4472, + "total_tokens": 100305, + "reasoning_tokens": 3416 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 38.66, + "avg_input_tokens": 74065.5, + "avg_output_tokens": 3886.2, + "avg_total_tokens": 77951.8, + "avg_turn_count": 13.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/desktop_template__contact_information.json b/task_results/desktop_template__contact_information.json index cd21ea0e..1c804cc0 100644 --- a/task_results/desktop_template__contact_information.json +++ b/task_results/desktop_template__contact_information.json @@ -2,6 +2,73 @@ "task": "desktop_template__contact_information", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 146.865305185318, + "agent_execution_time": 146.865305185318, + "token_usage": { + "input_tokens": 73597, + "output_tokens": 2268, + "total_tokens": 75865, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": true, + "execution_time": 674.874340057373, + "agent_execution_time": 674.874340057373, + "token_usage": { + "input_tokens": 30080, + "output_tokens": 17997, + "total_tokens": 48077, + "reasoning_tokens": 16458 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 701.8135657310486, + "agent_execution_time": 701.8135657310486, + "token_usage": { + "input_tokens": 37972, + "output_tokens": 16458, + "total_tokens": 54430, + "reasoning_tokens": 14966 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": true, + "execution_time": 907.022073507309, + "agent_execution_time": 907.022073507309, + "token_usage": { + "input_tokens": 52498, + "output_tokens": 24754, + "total_tokens": 77252, + "reasoning_tokens": 23124 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 607.64, + "avg_input_tokens": 48536.8, + "avg_output_tokens": 15369.2, + "avg_total_tokens": 63906.0, + "avg_turn_count": 8.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 161.22254180908203, - "agent_execution_time": 161.22254180908203, - "token_usage": { - "input_tokens": 54985, - "output_tokens": 2735, - "total_tokens": 57720, - "reasoning_tokens": null - }, - "turn_count": 10 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 161.22, - "avg_input_tokens": 54985.0, - "avg_output_tokens": 2735.0, - "avg_total_tokens": 57720.0, - "avg_turn_count": 10.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 87.12614560127258, + "agent_execution_time": 87.12614560127258, + "token_usage": { + "input_tokens": 61464, + "output_tokens": 2747, + "total_tokens": 64211, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 71.39080858230591, + "agent_execution_time": 71.39080858230591, + "token_usage": { + "input_tokens": 63198, + "output_tokens": 3322, + "total_tokens": 66520, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 75.84327030181885, + "agent_execution_time": 75.84327030181885, + "token_usage": { + "input_tokens": 105610, + "output_tokens": 3283, + "total_tokens": 108893, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 67.84210777282715, + "agent_execution_time": 67.84210777282715, + "token_usage": { + "input_tokens": 91793, + "output_tokens": 3251, + "total_tokens": 95044, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 75.55, + "avg_input_tokens": 80516.2, + "avg_output_tokens": 3150.8, + "avg_total_tokens": 83667.0, + "avg_turn_count": 12.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 44.054227113723755, + "agent_execution_time": 44.054227113723755, + "token_usage": { + "input_tokens": 91382, + "output_tokens": 2521, + "total_tokens": 93903, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": false, + "execution_time": 33.99298691749573, + "agent_execution_time": 33.99298691749573, + "token_usage": { + "input_tokens": 84386, + "output_tokens": 2527, + "total_tokens": 86913, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 44.054227113723755, + "agent_execution_time": 44.054227113723755, + "token_usage": { + "input_tokens": 91382, + "output_tokens": 2521, + "total_tokens": 93903, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 33.99298691749573, + "agent_execution_time": 33.99298691749573, + "token_usage": { + "input_tokens": 84386, + "output_tokens": 2527, + "total_tokens": 86913, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 39.02, + "avg_input_tokens": 87884.0, + "avg_output_tokens": 2524.0, + "avg_total_tokens": 90408.0, + "avg_turn_count": 21.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 149.86491012573242, + "agent_execution_time": 149.86491012573242, + "token_usage": { + "input_tokens": 67225, + "output_tokens": 2234, + "total_tokens": 69459, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 144.40228056907654, + "agent_execution_time": 144.40228056907654, + "token_usage": { + "input_tokens": 77463, + "output_tokens": 2134, + "total_tokens": 79597, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 11.23767614364624, + "agent_execution_time": 11.23767614364624, + "token_usage": { + "input_tokens": 3187, + "output_tokens": 209, + "total_tokens": 3396, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 141.92485570907593, + "agent_execution_time": 141.92485570907593, + "token_usage": { + "input_tokens": 53397, + "output_tokens": 2613, + "total_tokens": 56010, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 111.86, + "avg_input_tokens": 50318.0, + "avg_output_tokens": 1797.5, + "avg_total_tokens": 52115.5, + "avg_turn_count": 8.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 92.011953830719, + "agent_execution_time": 92.011953830719, + "token_usage": { + "input_tokens": 213694, + "output_tokens": 7413, + "total_tokens": 221107, + "reasoning_tokens": 5623 + }, + "turn_count": 31 + }, + { + "run": "run-2", + "success": true, + "execution_time": 54.116535902023315, + "agent_execution_time": 54.116535902023315, + "token_usage": { + "input_tokens": 137809, + "output_tokens": 6489, + "total_tokens": 144298, + "reasoning_tokens": 4809 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 78.22228336334229, + "agent_execution_time": 78.22228336334229, + "token_usage": { + "input_tokens": 229375, + "output_tokens": 5489, + "total_tokens": 234864, + "reasoning_tokens": 3272 + }, + "turn_count": 38 + }, + { + "run": "run-4", + "success": false, + "execution_time": 59.964751958847046, + "agent_execution_time": 59.964751958847046, + "token_usage": { + "input_tokens": 118938, + "output_tokens": 5836, + "total_tokens": 124774, + "reasoning_tokens": 4514 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 71.08, + "avg_input_tokens": 174954.0, + "avg_output_tokens": 6306.8, + "avg_total_tokens": 181260.8, + "avg_turn_count": 26.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/desktop_template__file_arrangement.json b/task_results/desktop_template__file_arrangement.json index edceeff2..cbd35360 100644 --- a/task_results/desktop_template__file_arrangement.json +++ b/task_results/desktop_template__file_arrangement.json @@ -2,6 +2,73 @@ "task": "desktop_template__file_arrangement", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 436.8873176574707, + "agent_execution_time": 436.8873176574707, + "token_usage": { + "input_tokens": 504678, + "output_tokens": 4914, + "total_tokens": 509592, + "reasoning_tokens": 0 + }, + "turn_count": 58 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1007.7418031692505, + "agent_execution_time": 1007.7418031692505, + "token_usage": { + "input_tokens": 80103, + "output_tokens": 27330, + "total_tokens": 107433, + "reasoning_tokens": 23666 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1110.845541715622, + "agent_execution_time": 1110.845541715622, + "token_usage": { + "input_tokens": 88256, + "output_tokens": 26333, + "total_tokens": 114589, + "reasoning_tokens": 22165 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1315.487462759018, + "agent_execution_time": 1315.487462759018, + "token_usage": { + "input_tokens": 80741, + "output_tokens": 36740, + "total_tokens": 117481, + "reasoning_tokens": 32638 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 967.74, + "avg_input_tokens": 188444.5, + "avg_output_tokens": 23829.2, + "avg_total_tokens": 212273.8, + "avg_turn_count": 22.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 581.3442401885986, - "agent_execution_time": 581.3442401885986, - "token_usage": { - "input_tokens": 587381, - "output_tokens": 9091, - "total_tokens": 596472, - "reasoning_tokens": null - }, - "turn_count": 53 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 581.34, - "avg_input_tokens": 587381.0, - "avg_output_tokens": 9091.0, - "avg_total_tokens": 596472.0, - "avg_turn_count": 53.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 159.62848925590515, + "agent_execution_time": 159.62848925590515, + "token_usage": { + "input_tokens": 142678, + "output_tokens": 5677, + "total_tokens": 148355, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 105.37904787063599, + "agent_execution_time": 105.37904787063599, + "token_usage": { + "input_tokens": 150584, + "output_tokens": 5759, + "total_tokens": 156343, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 73.0366370677948, + "agent_execution_time": 73.0366370677948, + "token_usage": { + "input_tokens": 93779, + "output_tokens": 4887, + "total_tokens": 98666, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 83.96731734275818, + "agent_execution_time": 83.96731734275818, + "token_usage": { + "input_tokens": 124307, + "output_tokens": 5346, + "total_tokens": 129653, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 105.5, + "avg_input_tokens": 127837.0, + "avg_output_tokens": 5417.2, + "avg_total_tokens": 133254.2, + "avg_turn_count": 14.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 120.52684664726257, + "agent_execution_time": 120.52684664726257, + "token_usage": { + "input_tokens": 225019, + "output_tokens": 3962, + "total_tokens": 228981, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-2", + "success": false, + "execution_time": 116.76458096504211, + "agent_execution_time": 116.76458096504211, + "token_usage": { + "input_tokens": 288160, + "output_tokens": 5002, + "total_tokens": 293162, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + { + "run": "run-3", + "success": false, + "execution_time": 120.52684664726257, + "agent_execution_time": 120.52684664726257, + "token_usage": { + "input_tokens": 225019, + "output_tokens": 3962, + "total_tokens": 228981, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-4", + "success": false, + "execution_time": 116.76458096504211, + "agent_execution_time": 116.76458096504211, + "token_usage": { + "input_tokens": 288160, + "output_tokens": 5002, + "total_tokens": 293162, + "reasoning_tokens": 0 + }, + "turn_count": 37 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 118.65, + "avg_input_tokens": 256589.5, + "avg_output_tokens": 4482.0, + "avg_total_tokens": 261071.5, + "avg_turn_count": 34.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 444.45455861091614, + "agent_execution_time": 444.45455861091614, + "token_usage": { + "input_tokens": 499497, + "output_tokens": 5039, + "total_tokens": 504536, + "reasoning_tokens": 0 + }, + "turn_count": 56 + }, + { + "run": "run-2", + "success": false, + "execution_time": 428.5998525619507, + "agent_execution_time": 428.5998525619507, + "token_usage": { + "input_tokens": 487278, + "output_tokens": 5025, + "total_tokens": 492303, + "reasoning_tokens": 0 + }, + "turn_count": 55 + }, + { + "run": "run-3", + "success": false, + "execution_time": 248.51806163787842, + "agent_execution_time": 248.51806163787842, + "token_usage": { + "input_tokens": 217679, + "output_tokens": 2867, + "total_tokens": 220546, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-4", + "success": false, + "execution_time": 223.56869792938232, + "agent_execution_time": 223.56869792938232, + "token_usage": { + "input_tokens": 107906, + "output_tokens": 5192, + "total_tokens": 113098, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 336.29, + "avg_input_tokens": 328090.0, + "avg_output_tokens": 4530.8, + "avg_total_tokens": 332620.8, + "avg_turn_count": 39.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 52.440882205963135, + "agent_execution_time": 52.440882205963135, + "token_usage": { + "input_tokens": 80407, + "output_tokens": 5584, + "total_tokens": 85991, + "reasoning_tokens": 2423 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 41.556867361068726, + "agent_execution_time": 41.556867361068726, + "token_usage": { + "input_tokens": 29973, + "output_tokens": 4927, + "total_tokens": 34900, + "reasoning_tokens": 2251 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": true, + "execution_time": 59.65416622161865, + "agent_execution_time": 59.65416622161865, + "token_usage": { + "input_tokens": 91228, + "output_tokens": 8705, + "total_tokens": 99933, + "reasoning_tokens": 4938 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 51.68978476524353, + "agent_execution_time": 51.68978476524353, + "token_usage": { + "input_tokens": 130077, + "output_tokens": 6596, + "total_tokens": 136673, + "reasoning_tokens": 3662 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 51.34, + "avg_input_tokens": 82921.2, + "avg_output_tokens": 6453.0, + "avg_total_tokens": 89374.2, + "avg_turn_count": 10.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/dvdrental__customer_analysis_fix.json b/task_results/dvdrental__customer_analysis_fix.json index b837e94d..73d0f05e 100644 --- a/task_results/dvdrental__customer_analysis_fix.json +++ b/task_results/dvdrental__customer_analysis_fix.json @@ -2,6 +2,73 @@ "task": "dvdrental__customer_analysis_fix", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 615.4603867530823, + "agent_execution_time": 615.4603867530823, + "token_usage": { + "input_tokens": 562080, + "output_tokens": 9415, + "total_tokens": 571495, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + { + "run": "run-2", + "success": false, + "execution_time": 436.2656843662262, + "agent_execution_time": 436.2656843662262, + "token_usage": { + "input_tokens": 112603, + "output_tokens": 2311, + "total_tokens": 114914, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 375.3437707424164, + "agent_execution_time": 375.3437707424164, + "token_usage": { + "input_tokens": 412950, + "output_tokens": 6024, + "total_tokens": 418974, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-4", + "success": false, + "execution_time": 77.14166474342346, + "agent_execution_time": 77.14166474342346, + "token_usage": { + "input_tokens": 3652, + "output_tokens": 1811, + "total_tokens": 5463, + "reasoning_tokens": 1780 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 376.05, + "avg_input_tokens": 272821.2, + "avg_output_tokens": 4890.2, + "avg_total_tokens": 277711.5, + "avg_turn_count": 20.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 225.17450332641602, - "agent_execution_time": 225.17450332641602, - "token_usage": { - "input_tokens": 56978, - "output_tokens": 4207, - "total_tokens": 61185, - "reasoning_tokens": null - }, - "turn_count": 11 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 225.17, - "avg_input_tokens": 56978.0, - "avg_output_tokens": 4207.0, - "avg_total_tokens": 61185.0, - "avg_turn_count": 11.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 135.74979424476624, + "agent_execution_time": 135.74979424476624, + "token_usage": { + "input_tokens": 263437, + "output_tokens": 6774, + "total_tokens": 270211, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": true, + "execution_time": 84.19785523414612, + "agent_execution_time": 84.19785523414612, + "token_usage": { + "input_tokens": 141888, + "output_tokens": 4135, + "total_tokens": 146023, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 122.44764375686646, + "agent_execution_time": 122.44764375686646, + "token_usage": { + "input_tokens": 242075, + "output_tokens": 5825, + "total_tokens": 247900, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 286.97572350502014, + "agent_execution_time": 286.97572350502014, + "token_usage": { + "input_tokens": 205801, + "output_tokens": 6004, + "total_tokens": 211805, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 157.34, + "avg_input_tokens": 213300.2, + "avg_output_tokens": 5684.5, + "avg_total_tokens": 218984.8, + "avg_turn_count": 16.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 206.21510243415833, + "agent_execution_time": 206.21510243415833, + "token_usage": { + "input_tokens": 245699, + "output_tokens": 6993, + "total_tokens": 252692, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 477.9997045993805, + "agent_execution_time": 477.9997045993805, + "token_usage": { + "input_tokens": 389086, + "output_tokens": 7594, + "total_tokens": 396680, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-3", + "success": false, + "execution_time": 435.499480009079, + "agent_execution_time": 435.499480009079, + "token_usage": { + "input_tokens": 321182, + "output_tokens": 7182, + "total_tokens": 328364, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-4", + "success": false, + "execution_time": 227.080819606781, + "agent_execution_time": 227.080819606781, + "token_usage": { + "input_tokens": 262025, + "output_tokens": 7421, + "total_tokens": 269446, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 336.7, + "avg_input_tokens": 304498.0, + "avg_output_tokens": 7297.5, + "avg_total_tokens": 311795.5, + "avg_turn_count": 23.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 304.07178139686584, + "agent_execution_time": 304.07178139686584, + "token_usage": { + "input_tokens": 258500, + "output_tokens": 5163, + "total_tokens": 263663, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 311.8955788612366, + "agent_execution_time": 311.8955788612366, + "token_usage": { + "input_tokens": 252658, + "output_tokens": 4708, + "total_tokens": 257366, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": false, + "execution_time": 344.65111804008484, + "agent_execution_time": 344.65111804008484, + "token_usage": { + "input_tokens": 284849, + "output_tokens": 5986, + "total_tokens": 290835, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.938684225082397, + "agent_execution_time": 8.938684225082397, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 242.39, + "avg_input_tokens": 199001.8, + "avg_output_tokens": 3964.2, + "avg_total_tokens": 202966.0, + "avg_turn_count": 16.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 114.13974523544312, + "agent_execution_time": 114.13974523544312, + "token_usage": { + "input_tokens": 259197, + "output_tokens": 12745, + "total_tokens": 271942, + "reasoning_tokens": 10039 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 104.39919996261597, + "agent_execution_time": 104.39919996261597, + "token_usage": { + "input_tokens": 571531, + "output_tokens": 9943, + "total_tokens": 581474, + "reasoning_tokens": 6295 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": true, + "execution_time": 111.87351703643799, + "agent_execution_time": 111.87351703643799, + "token_usage": { + "input_tokens": 258907, + "output_tokens": 12764, + "total_tokens": 271671, + "reasoning_tokens": 6973 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": true, + "execution_time": 114.15665888786316, + "agent_execution_time": 114.15665888786316, + "token_usage": { + "input_tokens": 234116, + "output_tokens": 10917, + "total_tokens": 245033, + "reasoning_tokens": 8227 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 111.14, + "avg_input_tokens": 330937.8, + "avg_output_tokens": 11592.2, + "avg_total_tokens": 342530.0, + "avg_turn_count": 21.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/dvdrental__customer_analytics_optimization.json b/task_results/dvdrental__customer_analytics_optimization.json index bebc1cfa..4ca181bb 100644 --- a/task_results/dvdrental__customer_analytics_optimization.json +++ b/task_results/dvdrental__customer_analytics_optimization.json @@ -2,6 +2,73 @@ "task": "dvdrental__customer_analytics_optimization", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 300.92593359947205, + "agent_execution_time": 300.92593359947205, + "token_usage": { + "input_tokens": 171637, + "output_tokens": 4732, + "total_tokens": 176369, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": true, + "execution_time": 270.64186120033264, + "agent_execution_time": 270.64186120033264, + "token_usage": { + "input_tokens": 190382, + "output_tokens": 4652, + "total_tokens": 195034, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": true, + "execution_time": 247.33644652366638, + "agent_execution_time": 247.33644652366638, + "token_usage": { + "input_tokens": 186229, + "output_tokens": 3891, + "total_tokens": 190120, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 210.79088759422302, + "agent_execution_time": 210.79088759422302, + "token_usage": { + "input_tokens": 12386, + "output_tokens": 5603, + "total_tokens": 17989, + "reasoning_tokens": 4398 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 257.42, + "avg_input_tokens": 140158.5, + "avg_output_tokens": 4719.5, + "avg_total_tokens": 144878.0, + "avg_turn_count": 14.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 263.71051263809204, - "agent_execution_time": 263.71051263809204, - "token_usage": { - "input_tokens": 362694, - "output_tokens": 4122, - "total_tokens": 366816, - "reasoning_tokens": null - }, - "turn_count": 14 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 263.71, - "avg_input_tokens": 362694.0, - "avg_output_tokens": 4122.0, - "avg_total_tokens": 366816.0, - "avg_turn_count": 14.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 112.24652767181396, + "agent_execution_time": 112.24652767181396, + "token_usage": { + "input_tokens": 220008, + "output_tokens": 5309, + "total_tokens": 225317, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": true, + "execution_time": 111.87647032737732, + "agent_execution_time": 111.87647032737732, + "token_usage": { + "input_tokens": 149256, + "output_tokens": 5916, + "total_tokens": 155172, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": true, + "execution_time": 167.59008932113647, + "agent_execution_time": 167.59008932113647, + "token_usage": { + "input_tokens": 254440, + "output_tokens": 7424, + "total_tokens": 261864, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": true, + "execution_time": 91.85426902770996, + "agent_execution_time": 91.85426902770996, + "token_usage": { + "input_tokens": 131884, + "output_tokens": 4949, + "total_tokens": 136833, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 120.89, + "avg_input_tokens": 188897.0, + "avg_output_tokens": 5899.5, + "avg_total_tokens": 194796.5, + "avg_turn_count": 14.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 139.83807396888733, + "agent_execution_time": 139.83807396888733, + "token_usage": { + "input_tokens": 135409, + "output_tokens": 4690, + "total_tokens": 140099, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 54.594237089157104, + "agent_execution_time": 54.594237089157104, + "token_usage": { + "input_tokens": 28523, + "output_tokens": 1076, + "total_tokens": 29599, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": true, + "execution_time": 189.83985376358032, + "agent_execution_time": 189.83985376358032, + "token_usage": { + "input_tokens": 229965, + "output_tokens": 3258, + "total_tokens": 233223, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": true, + "execution_time": 106.55440258979797, + "agent_execution_time": 106.55440258979797, + "token_usage": { + "input_tokens": 119927, + "output_tokens": 3940, + "total_tokens": 123867, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 122.71, + "avg_input_tokens": 128456.0, + "avg_output_tokens": 3241.0, + "avg_total_tokens": 131697.0, + "avg_turn_count": 12.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 239.05956530570984, + "agent_execution_time": 239.05956530570984, + "token_usage": { + "input_tokens": 135124, + "output_tokens": 4276, + "total_tokens": 139400, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": true, + "execution_time": 199.89162397384644, + "agent_execution_time": 199.89162397384644, + "token_usage": { + "input_tokens": 116717, + "output_tokens": 3357, + "total_tokens": 120074, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": true, + "execution_time": 152.3154890537262, + "agent_execution_time": 152.3154890537262, + "token_usage": { + "input_tokens": 86817, + "output_tokens": 2598, + "total_tokens": 89415, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.180521011352539, + "agent_execution_time": 8.180521011352539, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 149.86, + "avg_input_tokens": 84664.5, + "avg_output_tokens": 2557.8, + "avg_total_tokens": 87222.2, + "avg_turn_count": 9.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 60.74306917190552, + "agent_execution_time": 60.74306917190552, + "token_usage": { + "input_tokens": 118735, + "output_tokens": 6359, + "total_tokens": 125094, + "reasoning_tokens": 2249 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 58.639387369155884, + "agent_execution_time": 58.639387369155884, + "token_usage": { + "input_tokens": 45077, + "output_tokens": 5332, + "total_tokens": 50409, + "reasoning_tokens": 2368 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": true, + "execution_time": 51.7120943069458, + "agent_execution_time": 51.7120943069458, + "token_usage": { + "input_tokens": 257960, + "output_tokens": 5017, + "total_tokens": 262977, + "reasoning_tokens": 2595 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 39.97567844390869, + "agent_execution_time": 39.97567844390869, + "token_usage": { + "input_tokens": 10960, + "output_tokens": 3703, + "total_tokens": 14663, + "reasoning_tokens": 1325 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 52.77, + "avg_input_tokens": 108183.0, + "avg_output_tokens": 5102.8, + "avg_total_tokens": 113285.8, + "avg_turn_count": 8.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/dvdrental__film_inventory_management.json b/task_results/dvdrental__film_inventory_management.json index 291bd333..d15af41d 100644 --- a/task_results/dvdrental__film_inventory_management.json +++ b/task_results/dvdrental__film_inventory_management.json @@ -2,6 +2,73 @@ "task": "dvdrental__film_inventory_management", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 265.45637702941895, + "agent_execution_time": 265.45637702941895, + "token_usage": { + "input_tokens": 299262, + "output_tokens": 2949, + "total_tokens": 302211, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-2", + "success": true, + "execution_time": 264.9316096305847, + "agent_execution_time": 264.9316096305847, + "token_usage": { + "input_tokens": 385200, + "output_tokens": 3164, + "total_tokens": 388364, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-3", + "success": true, + "execution_time": 224.0705850124359, + "agent_execution_time": 224.0705850124359, + "token_usage": { + "input_tokens": 420685, + "output_tokens": 2560, + "total_tokens": 423245, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": true, + "execution_time": 495.70731568336487, + "agent_execution_time": 495.70731568336487, + "token_usage": { + "input_tokens": 11750, + "output_tokens": 13154, + "total_tokens": 24904, + "reasoning_tokens": 11981 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 312.54, + "avg_input_tokens": 279224.2, + "avg_output_tokens": 5456.8, + "avg_total_tokens": 284681.0, + "avg_turn_count": 22.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 201.0764443874359, - "agent_execution_time": 201.0764443874359, - "token_usage": { - "input_tokens": 57692, - "output_tokens": 3526, - "total_tokens": 61218, - "reasoning_tokens": null - }, - "turn_count": 13 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 201.08, - "avg_input_tokens": 57692.0, - "avg_output_tokens": 3526.0, - "avg_total_tokens": 61218.0, - "avg_turn_count": 13.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 103.67711782455444, + "agent_execution_time": 103.67711782455444, + "token_usage": { + "input_tokens": 441942, + "output_tokens": 3986, + "total_tokens": 445928, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": true, + "execution_time": 106.06457614898682, + "agent_execution_time": 106.06457614898682, + "token_usage": { + "input_tokens": 252654, + "output_tokens": 4305, + "total_tokens": 256959, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": true, + "execution_time": 99.84581923484802, + "agent_execution_time": 99.84581923484802, + "token_usage": { + "input_tokens": 179284, + "output_tokens": 3618, + "total_tokens": 182902, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": true, + "execution_time": 147.76059675216675, + "agent_execution_time": 147.76059675216675, + "token_usage": { + "input_tokens": 512847, + "output_tokens": 4572, + "total_tokens": 517419, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 114.34, + "avg_input_tokens": 346681.8, + "avg_output_tokens": 4120.2, + "avg_total_tokens": 350802.0, + "avg_turn_count": 18.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 172.88974380493164, + "agent_execution_time": 172.88974380493164, + "token_usage": { + "input_tokens": 597872, + "output_tokens": 4902, + "total_tokens": 602774, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": true, + "execution_time": 311.4297881126404, + "agent_execution_time": 311.4297881126404, + "token_usage": { + "input_tokens": 374222, + "output_tokens": 4855, + "total_tokens": 379077, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-3", + "success": true, + "execution_time": 247.373144865036, + "agent_execution_time": 247.373144865036, + "token_usage": { + "input_tokens": 467079, + "output_tokens": 5028, + "total_tokens": 472107, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-4", + "success": true, + "execution_time": 237.2136902809143, + "agent_execution_time": 237.2136902809143, + "token_usage": { + "input_tokens": 584711, + "output_tokens": 5865, + "total_tokens": 590576, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 242.23, + "avg_input_tokens": 505971.0, + "avg_output_tokens": 5162.5, + "avg_total_tokens": 511133.5, + "avg_turn_count": 26.25, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 200.07974195480347, + "agent_execution_time": 200.07974195480347, + "token_usage": { + "input_tokens": 227923, + "output_tokens": 2460, + "total_tokens": 230383, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": true, + "execution_time": 225.52838730812073, + "agent_execution_time": 225.52838730812073, + "token_usage": { + "input_tokens": 195296, + "output_tokens": 3270, + "total_tokens": 198566, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": true, + "execution_time": 199.81371188163757, + "agent_execution_time": 199.81371188163757, + "token_usage": { + "input_tokens": 202499, + "output_tokens": 2208, + "total_tokens": 204707, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.201359987258911, + "agent_execution_time": 8.201359987258911, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 158.41, + "avg_input_tokens": 156429.5, + "avg_output_tokens": 1984.5, + "avg_total_tokens": 158414.0, + "avg_turn_count": 18.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 26.48506808280945, + "agent_execution_time": 26.48506808280945, + "token_usage": { + "input_tokens": 49443, + "output_tokens": 3283, + "total_tokens": 52726, + "reasoning_tokens": 2477 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": true, + "execution_time": 130.48151683807373, + "agent_execution_time": 130.48151683807373, + "token_usage": { + "input_tokens": 18333, + "output_tokens": 5529, + "total_tokens": 23862, + "reasoning_tokens": 4595 + }, + "turn_count": 3 + }, + { + "run": "run-3", + "success": true, + "execution_time": 37.734288930892944, + "agent_execution_time": 37.734288930892944, + "token_usage": { + "input_tokens": 44486, + "output_tokens": 2901, + "total_tokens": 47387, + "reasoning_tokens": 2003 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": true, + "execution_time": 25.89009380340576, + "agent_execution_time": 25.89009380340576, + "token_usage": { + "input_tokens": 33609, + "output_tokens": 2442, + "total_tokens": 36051, + "reasoning_tokens": 1533 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 55.15, + "avg_input_tokens": 36467.8, + "avg_output_tokens": 3538.8, + "avg_total_tokens": 40006.5, + "avg_turn_count": 8.25, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/easyr1__advanced_branch_strategy.json b/task_results/easyr1__advanced_branch_strategy.json index dc33385f..08aa1efb 100644 --- a/task_results/easyr1__advanced_branch_strategy.json +++ b/task_results/easyr1__advanced_branch_strategy.json @@ -2,6 +2,73 @@ "task": "easyr1__advanced_branch_strategy", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 133.41261291503906, + "agent_execution_time": 133.41261291503906, + "token_usage": { + "input_tokens": 236001, + "output_tokens": 1385, + "total_tokens": 237386, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": true, + "execution_time": 657.4795656204224, + "agent_execution_time": 657.4795656204224, + "token_usage": { + "input_tokens": 206206, + "output_tokens": 16956, + "total_tokens": 223162, + "reasoning_tokens": 15853 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": true, + "execution_time": 969.739565372467, + "agent_execution_time": 969.739565372467, + "token_usage": { + "input_tokens": 281358, + "output_tokens": 20893, + "total_tokens": 302251, + "reasoning_tokens": 19803 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 43.79906630516052, + "agent_execution_time": 43.79906630516052, + "token_usage": { + "input_tokens": 23930, + "output_tokens": 1206, + "total_tokens": 25136, + "reasoning_tokens": 315 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 451.11, + "avg_input_tokens": 186873.8, + "avg_output_tokens": 10110.0, + "avg_total_tokens": 196983.8, + "avg_turn_count": 8.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 198.91717386245728, - "agent_execution_time": 198.91717386245728, - "token_usage": { - "input_tokens": 235867, - "output_tokens": 2372, - "total_tokens": 238239, - "reasoning_tokens": null - }, - "turn_count": 10 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 198.92, - "avg_input_tokens": 235867.0, - "avg_output_tokens": 2372.0, - "avg_total_tokens": 238239.0, - "avg_turn_count": 10.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 94.20567393302917, + "agent_execution_time": 94.20567393302917, + "token_usage": { + "input_tokens": 243093, + "output_tokens": 2513, + "total_tokens": 245606, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": true, + "execution_time": 85.33322143554688, + "agent_execution_time": 85.33322143554688, + "token_usage": { + "input_tokens": 258074, + "output_tokens": 2653, + "total_tokens": 260727, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": true, + "execution_time": 69.08874940872192, + "agent_execution_time": 69.08874940872192, + "token_usage": { + "input_tokens": 163053, + "output_tokens": 2039, + "total_tokens": 165092, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": true, + "execution_time": 65.48941016197205, + "agent_execution_time": 65.48941016197205, + "token_usage": { + "input_tokens": 180247, + "output_tokens": 1960, + "total_tokens": 182207, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 78.53, + "avg_input_tokens": 211116.8, + "avg_output_tokens": 2291.2, + "avg_total_tokens": 213408.0, + "avg_turn_count": 9.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 139.20880770683289, + "agent_execution_time": 139.20880770683289, + "token_usage": { + "input_tokens": 205868, + "output_tokens": 2715, + "total_tokens": 208583, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": true, + "execution_time": 105.78681325912476, + "agent_execution_time": 105.78681325912476, + "token_usage": { + "input_tokens": 223553, + "output_tokens": 1659, + "total_tokens": 225212, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": true, + "execution_time": 116.28803777694702, + "agent_execution_time": 116.28803777694702, + "token_usage": { + "input_tokens": 342597, + "output_tokens": 1994, + "total_tokens": 344591, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": true, + "execution_time": 134.20125770568848, + "agent_execution_time": 134.20125770568848, + "token_usage": { + "input_tokens": 286283, + "output_tokens": 3335, + "total_tokens": 289618, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 123.87, + "avg_input_tokens": 264575.2, + "avg_output_tokens": 2425.8, + "avg_total_tokens": 267001.0, + "avg_turn_count": 19.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 94.89696407318115, + "agent_execution_time": 94.89696407318115, + "token_usage": { + "input_tokens": 145445, + "output_tokens": 1014, + "total_tokens": 146459, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.81865382194519, + "agent_execution_time": 10.81865382194519, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 55.28738832473755, + "agent_execution_time": 55.28738832473755, + "token_usage": { + "input_tokens": 23930, + "output_tokens": 838, + "total_tokens": 24768, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 86.20849013328552, + "agent_execution_time": 86.20849013328552, + "token_usage": { + "input_tokens": 77371, + "output_tokens": 1566, + "total_tokens": 78937, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 61.8, + "avg_input_tokens": 61686.5, + "avg_output_tokens": 854.5, + "avg_total_tokens": 62541.0, + "avg_turn_count": 3.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 57.045555114746094, + "agent_execution_time": 57.045555114746094, + "token_usage": { + "input_tokens": 82074, + "output_tokens": 2965, + "total_tokens": 85039, + "reasoning_tokens": 2011 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 67.60392880439758, + "agent_execution_time": 67.60392880439758, + "token_usage": { + "input_tokens": 95293, + "output_tokens": 6425, + "total_tokens": 101718, + "reasoning_tokens": 5102 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": true, + "execution_time": 58.84398698806763, + "agent_execution_time": 58.84398698806763, + "token_usage": { + "input_tokens": 181063, + "output_tokens": 4678, + "total_tokens": 185741, + "reasoning_tokens": 3850 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": true, + "execution_time": 57.27000713348389, + "agent_execution_time": 57.27000713348389, + "token_usage": { + "input_tokens": 163654, + "output_tokens": 2875, + "total_tokens": 166529, + "reasoning_tokens": 2034 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 60.19, + "avg_input_tokens": 130521.0, + "avg_output_tokens": 4235.8, + "avg_total_tokens": 134756.8, + "avg_turn_count": 7.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/easyr1__config_parameter_audit.json b/task_results/easyr1__config_parameter_audit.json index 888d7bba..eb49ada1 100644 --- a/task_results/easyr1__config_parameter_audit.json +++ b/task_results/easyr1__config_parameter_audit.json @@ -2,6 +2,73 @@ "task": "easyr1__config_parameter_audit", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 313.7692756652832, + "agent_execution_time": 313.7692756652832, + "token_usage": { + "input_tokens": 1119193, + "output_tokens": 2255, + "total_tokens": 1121448, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": false, + "execution_time": 983.0928440093994, + "agent_execution_time": 983.0928440093994, + "token_usage": { + "input_tokens": 257270, + "output_tokens": 24271, + "total_tokens": 281541, + "reasoning_tokens": 23047 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 789.3945915699005, + "agent_execution_time": 789.3945915699005, + "token_usage": { + "input_tokens": 304602, + "output_tokens": 16554, + "total_tokens": 321156, + "reasoning_tokens": 16101 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 716.6567142009735, + "agent_execution_time": 716.6567142009735, + "token_usage": { + "input_tokens": 287608, + "output_tokens": 19051, + "total_tokens": 306659, + "reasoning_tokens": 18659 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 700.73, + "avg_input_tokens": 492168.2, + "avg_output_tokens": 15532.8, + "avg_total_tokens": 507701.0, + "avg_turn_count": 10.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 577.8364017009735, - "agent_execution_time": 577.8364017009735, - "token_usage": { - "input_tokens": 3295742, - "output_tokens": 2949, - "total_tokens": 3298691, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 577.84, - "avg_input_tokens": 3295742.0, - "avg_output_tokens": 2949.0, - "avg_total_tokens": 3298691.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 240.67952251434326, + "agent_execution_time": 240.67952251434326, + "token_usage": { + "input_tokens": 1607883, + "output_tokens": 3981, + "total_tokens": 1611864, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 113.6022355556488, + "agent_execution_time": 113.6022355556488, + "token_usage": { + "input_tokens": 1198253, + "output_tokens": 3211, + "total_tokens": 1201464, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 210.83103489875793, + "agent_execution_time": 210.83103489875793, + "token_usage": { + "input_tokens": 926443, + "output_tokens": 3065, + "total_tokens": 929508, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": false, + "execution_time": 116.27509450912476, + "agent_execution_time": 116.27509450912476, + "token_usage": { + "input_tokens": 920720, + "output_tokens": 2926, + "total_tokens": 923646, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 170.35, + "avg_input_tokens": 1163324.8, + "avg_output_tokens": 3295.8, + "avg_total_tokens": 1166620.5, + "avg_turn_count": 14.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 356.9303925037384, + "agent_execution_time": 356.9303925037384, + "token_usage": { + "input_tokens": 1516433, + "output_tokens": 4490, + "total_tokens": 1520923, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-2", + "success": false, + "execution_time": 256.49879240989685, + "agent_execution_time": 256.49879240989685, + "token_usage": { + "input_tokens": 1576366, + "output_tokens": 1758, + "total_tokens": 1578124, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 249.4454951286316, + "agent_execution_time": 249.4454951286316, + "token_usage": { + "input_tokens": 1827788, + "output_tokens": 2158, + "total_tokens": 1829946, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-4", + "success": false, + "execution_time": 191.3074803352356, + "agent_execution_time": 191.3074803352356, + "token_usage": { + "input_tokens": 1312255, + "output_tokens": 3461, + "total_tokens": 1315716, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 263.55, + "avg_input_tokens": 1558210.5, + "avg_output_tokens": 2966.8, + "avg_total_tokens": 1561177.2, + "avg_turn_count": 25.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 252.89350628852844, + "agent_execution_time": 252.89350628852844, + "token_usage": { + "input_tokens": 977681, + "output_tokens": 1940, + "total_tokens": 979621, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.738064527511597, + "agent_execution_time": 10.738064527511597, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 727.9544639587402, + "agent_execution_time": 727.9544639587402, + "token_usage": { + "input_tokens": 1190015, + "output_tokens": 1609, + "total_tokens": 1191624, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 607.4092285633087, + "agent_execution_time": 607.4092285633087, + "token_usage": { + "input_tokens": 1315747, + "output_tokens": 2155, + "total_tokens": 1317902, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 399.75, + "avg_input_tokens": 870860.8, + "avg_output_tokens": 1426.0, + "avg_total_tokens": 872286.8, + "avg_turn_count": 13.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 93.72474551200867, + "agent_execution_time": 93.72474551200867, + "token_usage": { + "input_tokens": 527993, + "output_tokens": 6246, + "total_tokens": 534239, + "reasoning_tokens": 4365 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1240.415374994278, + "agent_execution_time": 1240.415374994278, + "token_usage": { + "input_tokens": 3755379, + "output_tokens": 111412, + "total_tokens": 3866791, + "reasoning_tokens": 3063 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 101.83688473701477, + "agent_execution_time": 101.83688473701477, + "token_usage": { + "input_tokens": 239032, + "output_tokens": 7199, + "total_tokens": 246231, + "reasoning_tokens": 2046 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 66.84354496002197, + "agent_execution_time": 66.84354496002197, + "token_usage": { + "input_tokens": 373114, + "output_tokens": 3853, + "total_tokens": 376967, + "reasoning_tokens": 2718 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 375.71, + "avg_input_tokens": 1223879.5, + "avg_output_tokens": 32177.5, + "avg_total_tokens": 1256057.0, + "avg_turn_count": 16.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/easyr1__performance_regression_investigation.json b/task_results/easyr1__performance_regression_investigation.json index a396cca1..f3726626 100644 --- a/task_results/easyr1__performance_regression_investigation.json +++ b/task_results/easyr1__performance_regression_investigation.json @@ -2,6 +2,73 @@ "task": "easyr1__performance_regression_investigation", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 190.212881565094, + "agent_execution_time": 190.212881565094, + "token_usage": { + "input_tokens": 292344, + "output_tokens": 2611, + "total_tokens": 294955, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": true, + "execution_time": 508.85288095474243, + "agent_execution_time": 508.85288095474243, + "token_usage": { + "input_tokens": 90096, + "output_tokens": 12249, + "total_tokens": 102345, + "reasoning_tokens": 10191 + }, + "turn_count": 3 + }, + { + "run": "run-3", + "success": true, + "execution_time": 680.2623610496521, + "agent_execution_time": 680.2623610496521, + "token_usage": { + "input_tokens": 129262, + "output_tokens": 15538, + "total_tokens": 144800, + "reasoning_tokens": 13387 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 478.04176664352417, + "agent_execution_time": 478.04176664352417, + "token_usage": { + "input_tokens": 109780, + "output_tokens": 12801, + "total_tokens": 122581, + "reasoning_tokens": 11140 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 464.34, + "avg_input_tokens": 155370.5, + "avg_output_tokens": 10799.8, + "avg_total_tokens": 166170.2, + "avg_turn_count": 5.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 390.20820927619934, - "agent_execution_time": 390.20820927619934, - "token_usage": { - "input_tokens": 450998, - "output_tokens": 5172, - "total_tokens": 456170, - "reasoning_tokens": null - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 390.21, - "avg_input_tokens": 450998.0, - "avg_output_tokens": 5172.0, - "avg_total_tokens": 456170.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 125.40516710281372, + "agent_execution_time": 125.40516710281372, + "token_usage": { + "input_tokens": 259202, + "output_tokens": 4610, + "total_tokens": 263812, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": true, + "execution_time": 102.49504208564758, + "agent_execution_time": 102.49504208564758, + "token_usage": { + "input_tokens": 261044, + "output_tokens": 4027, + "total_tokens": 265071, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": true, + "execution_time": 157.9911093711853, + "agent_execution_time": 157.9911093711853, + "token_usage": { + "input_tokens": 420174, + "output_tokens": 6557, + "total_tokens": 426731, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": true, + "execution_time": 169.86899995803833, + "agent_execution_time": 169.86899995803833, + "token_usage": { + "input_tokens": 425687, + "output_tokens": 6912, + "total_tokens": 432599, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 138.94, + "avg_input_tokens": 341526.8, + "avg_output_tokens": 5526.5, + "avg_total_tokens": 347053.2, + "avg_turn_count": 10.25, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 181.65913248062134, + "agent_execution_time": 181.65913248062134, + "token_usage": { + "input_tokens": 285436, + "output_tokens": 3618, + "total_tokens": 289054, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": true, + "execution_time": 198.75509119033813, + "agent_execution_time": 198.75509119033813, + "token_usage": { + "input_tokens": 331025, + "output_tokens": 3183, + "total_tokens": 334208, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": true, + "execution_time": 96.5533332824707, + "agent_execution_time": 96.5533332824707, + "token_usage": { + "input_tokens": 252968, + "output_tokens": 2576, + "total_tokens": 255544, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": true, + "execution_time": 178.24584770202637, + "agent_execution_time": 178.24584770202637, + "token_usage": { + "input_tokens": 368401, + "output_tokens": 5183, + "total_tokens": 373584, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 163.8, + "avg_input_tokens": 309457.5, + "avg_output_tokens": 3640.0, + "avg_total_tokens": 313097.5, + "avg_turn_count": 19.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 216.1938133239746, + "agent_execution_time": 216.1938133239746, + "token_usage": { + "input_tokens": 366322, + "output_tokens": 2752, + "total_tokens": 369074, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.795209407806396, + "agent_execution_time": 10.795209407806396, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 212.5976676940918, + "agent_execution_time": 212.5976676940918, + "token_usage": { + "input_tokens": 163507, + "output_tokens": 3188, + "total_tokens": 166695, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": true, + "execution_time": 225.86718153953552, + "agent_execution_time": 225.86718153953552, + "token_usage": { + "input_tokens": 193134, + "output_tokens": 3990, + "total_tokens": 197124, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 166.36, + "avg_input_tokens": 180740.8, + "avg_output_tokens": 2482.5, + "avg_total_tokens": 183223.2, + "avg_turn_count": 7.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 44.879680156707764, + "agent_execution_time": 44.879680156707764, + "token_usage": { + "input_tokens": 56262, + "output_tokens": 3113, + "total_tokens": 59375, + "reasoning_tokens": 1438 + }, + "turn_count": 3 + }, + { + "run": "run-2", + "success": true, + "execution_time": 75.66387295722961, + "agent_execution_time": 75.66387295722961, + "token_usage": { + "input_tokens": 79241, + "output_tokens": 5587, + "total_tokens": 84828, + "reasoning_tokens": 4018 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 332.8027970790863, + "agent_execution_time": 332.8027970790863, + "token_usage": { + "input_tokens": 290569, + "output_tokens": 13855, + "total_tokens": 304424, + "reasoning_tokens": 2432 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": true, + "execution_time": 39.77624320983887, + "agent_execution_time": 39.77624320983887, + "token_usage": { + "input_tokens": 99935, + "output_tokens": 3485, + "total_tokens": 103420, + "reasoning_tokens": 1919 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 123.28, + "avg_input_tokens": 131501.8, + "avg_output_tokens": 6510.0, + "avg_total_tokens": 138011.8, + "avg_turn_count": 5.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/easyr1__qwen3_issue_management.json b/task_results/easyr1__qwen3_issue_management.json index 565fbbd6..6a7c291b 100644 --- a/task_results/easyr1__qwen3_issue_management.json +++ b/task_results/easyr1__qwen3_issue_management.json @@ -2,6 +2,73 @@ "task": "easyr1__qwen3_issue_management", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 133.20124340057373, + "agent_execution_time": 133.20124340057373, + "token_usage": { + "input_tokens": 365355, + "output_tokens": 1200, + "total_tokens": 366555, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 253.49759078025818, + "agent_execution_time": 253.49759078025818, + "token_usage": { + "input_tokens": 75866, + "output_tokens": 5494, + "total_tokens": 81360, + "reasoning_tokens": 4984 + }, + "turn_count": 3 + }, + { + "run": "run-3", + "success": false, + "execution_time": 778.1814601421356, + "agent_execution_time": 778.1814601421356, + "token_usage": { + "input_tokens": 222374, + "output_tokens": 17335, + "total_tokens": 239709, + "reasoning_tokens": 16261 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 433.4517014026642, + "agent_execution_time": 433.4517014026642, + "token_usage": { + "input_tokens": 164397, + "output_tokens": 11797, + "total_tokens": 176194, + "reasoning_tokens": 11095 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 399.58, + "avg_input_tokens": 206998.0, + "avg_output_tokens": 8956.5, + "avg_total_tokens": 215954.5, + "avg_turn_count": 6.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 288.1276867389679, - "agent_execution_time": 288.1276867389679, - "token_usage": { - "input_tokens": 653543, - "output_tokens": 2775, - "total_tokens": 656318, - "reasoning_tokens": null - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 288.13, - "avg_input_tokens": 653543.0, - "avg_output_tokens": 2775.0, - "avg_total_tokens": 656318.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 48.81993341445923, + "agent_execution_time": 48.81993341445923, + "token_usage": { + "input_tokens": 136452, + "output_tokens": 1438, + "total_tokens": 137890, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 58.35715985298157, + "agent_execution_time": 58.35715985298157, + "token_usage": { + "input_tokens": 172705, + "output_tokens": 1805, + "total_tokens": 174510, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 63.62937140464783, + "agent_execution_time": 63.62937140464783, + "token_usage": { + "input_tokens": 239392, + "output_tokens": 1478, + "total_tokens": 240870, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 48.61015057563782, + "agent_execution_time": 48.61015057563782, + "token_usage": { + "input_tokens": 145628, + "output_tokens": 1428, + "total_tokens": 147056, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 54.85, + "avg_input_tokens": 173544.2, + "avg_output_tokens": 1537.2, + "avg_total_tokens": 175081.5, + "avg_turn_count": 6.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 96.23404240608215, + "agent_execution_time": 96.23404240608215, + "token_usage": { + "input_tokens": 228550, + "output_tokens": 2100, + "total_tokens": 230650, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 41.0689959526062, + "agent_execution_time": 41.0689959526062, + "token_usage": { + "input_tokens": 90512, + "output_tokens": 681, + "total_tokens": 91193, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 60.37042284011841, + "agent_execution_time": 60.37042284011841, + "token_usage": { + "input_tokens": 74504, + "output_tokens": 784, + "total_tokens": 75288, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 40.03966999053955, + "agent_execution_time": 40.03966999053955, + "token_usage": { + "input_tokens": 89233, + "output_tokens": 1324, + "total_tokens": 90557, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 59.43, + "avg_input_tokens": 120699.8, + "avg_output_tokens": 1222.2, + "avg_total_tokens": 121922.0, + "avg_turn_count": 8.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 96.25566291809082, + "agent_execution_time": 96.25566291809082, + "token_usage": { + "input_tokens": 177209, + "output_tokens": 948, + "total_tokens": 178157, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 92.95255088806152, + "agent_execution_time": 92.95255088806152, + "token_usage": { + "input_tokens": 162333, + "output_tokens": 872, + "total_tokens": 163205, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 99.37275528907776, + "agent_execution_time": 99.37275528907776, + "token_usage": { + "input_tokens": 127991, + "output_tokens": 665, + "total_tokens": 128656, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 50.10952401161194, + "agent_execution_time": 50.10952401161194, + "token_usage": { + "input_tokens": 101975, + "output_tokens": 560, + "total_tokens": 102535, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 84.67, + "avg_input_tokens": 142377.0, + "avg_output_tokens": 761.2, + "avg_total_tokens": 143138.2, + "avg_turn_count": 5.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 65.41552782058716, + "agent_execution_time": 65.41552782058716, + "token_usage": { + "input_tokens": 1028466, + "output_tokens": 2069, + "total_tokens": 1030535, + "reasoning_tokens": 1127 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 80.247314453125, + "agent_execution_time": 80.247314453125, + "token_usage": { + "input_tokens": 1530229, + "output_tokens": 1578, + "total_tokens": 1531807, + "reasoning_tokens": 1290 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 96.00932621955872, + "agent_execution_time": 96.00932621955872, + "token_usage": { + "input_tokens": 1751625, + "output_tokens": 3825, + "total_tokens": 1755450, + "reasoning_tokens": 2569 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 63.5764684677124, + "agent_execution_time": 63.5764684677124, + "token_usage": { + "input_tokens": 237245, + "output_tokens": 3853, + "total_tokens": 241098, + "reasoning_tokens": 2441 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 76.31, + "avg_input_tokens": 1136891.2, + "avg_output_tokens": 2831.2, + "avg_total_tokens": 1139722.5, + "avg_turn_count": 9.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/employees__employee_demographics_report.json b/task_results/employees__employee_demographics_report.json index efc7a5c9..e03447f4 100644 --- a/task_results/employees__employee_demographics_report.json +++ b/task_results/employees__employee_demographics_report.json @@ -2,6 +2,73 @@ "task": "employees__employee_demographics_report", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 373.7859036922455, + "agent_execution_time": 373.7859036922455, + "token_usage": { + "input_tokens": 271794, + "output_tokens": 4959, + "total_tokens": 276753, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-2", + "success": false, + "execution_time": 345.913902759552, + "agent_execution_time": 345.913902759552, + "token_usage": { + "input_tokens": 232934, + "output_tokens": 5260, + "total_tokens": 238194, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-3", + "success": true, + "execution_time": 249.38017582893372, + "agent_execution_time": 249.38017582893372, + "token_usage": { + "input_tokens": 177824, + "output_tokens": 3398, + "total_tokens": 181222, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-4", + "success": true, + "execution_time": 707.6514430046082, + "agent_execution_time": 707.6514430046082, + "token_usage": { + "input_tokens": 60163, + "output_tokens": 18420, + "total_tokens": 78583, + "reasoning_tokens": 16586 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 419.18, + "avg_input_tokens": 185678.8, + "avg_output_tokens": 8009.2, + "avg_total_tokens": 193688.0, + "avg_turn_count": 23.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 518.9978232383728, - "agent_execution_time": 518.9978232383728, - "token_usage": { - "input_tokens": 234688, - "output_tokens": 9902, - "total_tokens": 244590, - "reasoning_tokens": null - }, - "turn_count": 29 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 519.0, - "avg_input_tokens": 234688.0, - "avg_output_tokens": 9902.0, - "avg_total_tokens": 244590.0, - "avg_turn_count": 29.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 110.05146908760071, + "agent_execution_time": 110.05146908760071, + "token_usage": { + "input_tokens": 136728, + "output_tokens": 5736, + "total_tokens": 142464, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 132.65369296073914, + "agent_execution_time": 132.65369296073914, + "token_usage": { + "input_tokens": 188251, + "output_tokens": 5395, + "total_tokens": 193646, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": true, + "execution_time": 135.7118330001831, + "agent_execution_time": 135.7118330001831, + "token_usage": { + "input_tokens": 209718, + "output_tokens": 6943, + "total_tokens": 216661, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": true, + "execution_time": 182.46520376205444, + "agent_execution_time": 182.46520376205444, + "token_usage": { + "input_tokens": 277131, + "output_tokens": 8321, + "total_tokens": 285452, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 140.22, + "avg_input_tokens": 202957.0, + "avg_output_tokens": 6598.8, + "avg_total_tokens": 209555.8, + "avg_turn_count": 18.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 211.34916234016418, + "agent_execution_time": 211.34916234016418, + "token_usage": { + "input_tokens": 177350, + "output_tokens": 4606, + "total_tokens": 181956, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 406.7878062725067, + "agent_execution_time": 406.7878062725067, + "token_usage": { + "input_tokens": 187873, + "output_tokens": 4188, + "total_tokens": 192061, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": true, + "execution_time": 232.28180265426636, + "agent_execution_time": 232.28180265426636, + "token_usage": { + "input_tokens": 205272, + "output_tokens": 4174, + "total_tokens": 209446, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": false, + "execution_time": 172.16791248321533, + "agent_execution_time": 172.16791248321533, + "token_usage": { + "input_tokens": 284662, + "output_tokens": 4580, + "total_tokens": 289242, + "reasoning_tokens": 0 + }, + "turn_count": 30 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 255.65, + "avg_input_tokens": 213789.2, + "avg_output_tokens": 4387.0, + "avg_total_tokens": 218176.2, + "avg_turn_count": 26.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 177.2384054660797, + "agent_execution_time": 177.2384054660797, + "token_usage": { + "input_tokens": 126415, + "output_tokens": 2374, + "total_tokens": 128789, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 221.10795068740845, + "agent_execution_time": 221.10795068740845, + "token_usage": { + "input_tokens": 137479, + "output_tokens": 3172, + "total_tokens": 140651, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 256.72964096069336, + "agent_execution_time": 256.72964096069336, + "token_usage": { + "input_tokens": 152656, + "output_tokens": 3932, + "total_tokens": 156588, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.208264112472534, + "agent_execution_time": 8.208264112472534, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 165.82, + "avg_input_tokens": 104137.5, + "avg_output_tokens": 2369.5, + "avg_total_tokens": 106507.0, + "avg_turn_count": 15.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 37.23557090759277, + "agent_execution_time": 37.23557090759277, + "token_usage": { + "input_tokens": 65053, + "output_tokens": 3774, + "total_tokens": 68827, + "reasoning_tokens": 2399 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 52.087852478027344, + "agent_execution_time": 52.087852478027344, + "token_usage": { + "input_tokens": 107073, + "output_tokens": 6365, + "total_tokens": 113438, + "reasoning_tokens": 4397 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 47.810582876205444, + "agent_execution_time": 47.810582876205444, + "token_usage": { + "input_tokens": 60858, + "output_tokens": 4153, + "total_tokens": 65011, + "reasoning_tokens": 1874 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": true, + "execution_time": 60.068111419677734, + "agent_execution_time": 60.068111419677734, + "token_usage": { + "input_tokens": 84425, + "output_tokens": 4111, + "total_tokens": 88536, + "reasoning_tokens": 2115 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 49.3, + "avg_input_tokens": 79352.2, + "avg_output_tokens": 4600.8, + "avg_total_tokens": 83953.0, + "avg_turn_count": 12.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/employees__employee_performance_analysis.json b/task_results/employees__employee_performance_analysis.json index 3f091573..53cdf4f7 100644 --- a/task_results/employees__employee_performance_analysis.json +++ b/task_results/employees__employee_performance_analysis.json @@ -2,6 +2,73 @@ "task": "employees__employee_performance_analysis", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 219.51662135124207, + "agent_execution_time": 219.51662135124207, + "token_usage": { + "input_tokens": 132930, + "output_tokens": 2862, + "total_tokens": 135792, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": true, + "execution_time": 408.2539646625519, + "agent_execution_time": 408.2539646625519, + "token_usage": { + "input_tokens": 280076, + "output_tokens": 6444, + "total_tokens": 286520, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-3", + "success": true, + "execution_time": 177.42112493515015, + "agent_execution_time": 177.42112493515015, + "token_usage": { + "input_tokens": 119897, + "output_tokens": 2208, + "total_tokens": 122105, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 174.39799976348877, + "agent_execution_time": 174.39799976348877, + "token_usage": { + "input_tokens": 24302, + "output_tokens": 3950, + "total_tokens": 28252, + "reasoning_tokens": 3695 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 244.9, + "avg_input_tokens": 139301.2, + "avg_output_tokens": 3866.0, + "avg_total_tokens": 143167.2, + "avg_turn_count": 17.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 658.864099740982, - "agent_execution_time": 658.864099740982, - "token_usage": { - "input_tokens": 313303, - "output_tokens": 12804, - "total_tokens": 326107, - "reasoning_tokens": null - }, - "turn_count": 34 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 658.86, - "avg_input_tokens": 313303.0, - "avg_output_tokens": 12804.0, - "avg_total_tokens": 326107.0, - "avg_turn_count": 34.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 349.44184613227844, + "agent_execution_time": 349.44184613227844, + "token_usage": { + "input_tokens": 309034, + "output_tokens": 8445, + "total_tokens": 317479, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 118.50761294364929, + "agent_execution_time": 118.50761294364929, + "token_usage": { + "input_tokens": 244312, + "output_tokens": 6021, + "total_tokens": 250333, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": false, + "execution_time": 153.22221660614014, + "agent_execution_time": 153.22221660614014, + "token_usage": { + "input_tokens": 373968, + "output_tokens": 7602, + "total_tokens": 381570, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 162.31071543693542, + "agent_execution_time": 162.31071543693542, + "token_usage": { + "input_tokens": 223674, + "output_tokens": 6990, + "total_tokens": 230664, + "reasoning_tokens": 0 + }, + "turn_count": 16 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 195.87, + "avg_input_tokens": 287747.0, + "avg_output_tokens": 7264.5, + "avg_total_tokens": 295011.5, + "avg_turn_count": 18.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 184.1734356880188, + "agent_execution_time": 184.1734356880188, + "token_usage": { + "input_tokens": 273991, + "output_tokens": 5706, + "total_tokens": 279697, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-2", + "success": false, + "execution_time": 223.809636592865, + "agent_execution_time": 223.809636592865, + "token_usage": { + "input_tokens": 140996, + "output_tokens": 3991, + "total_tokens": 144987, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 131.49172067642212, + "agent_execution_time": 131.49172067642212, + "token_usage": { + "input_tokens": 141717, + "output_tokens": 4077, + "total_tokens": 145794, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 141.07456827163696, + "agent_execution_time": 141.07456827163696, + "token_usage": { + "input_tokens": 192246, + "output_tokens": 4091, + "total_tokens": 196337, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 170.14, + "avg_input_tokens": 187237.5, + "avg_output_tokens": 4466.2, + "avg_total_tokens": 191703.8, + "avg_turn_count": 22.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 209.56854391098022, + "agent_execution_time": 209.56854391098022, + "token_usage": { + "input_tokens": 129164, + "output_tokens": 3086, + "total_tokens": 132250, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 330.1731457710266, + "agent_execution_time": 330.1731457710266, + "token_usage": { + "input_tokens": 191625, + "output_tokens": 5463, + "total_tokens": 197088, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": true, + "execution_time": 191.89989256858826, + "agent_execution_time": 191.89989256858826, + "token_usage": { + "input_tokens": 122804, + "output_tokens": 2684, + "total_tokens": 125488, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.182923793792725, + "agent_execution_time": 8.182923793792725, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 184.96, + "avg_input_tokens": 110898.2, + "avg_output_tokens": 2808.2, + "avg_total_tokens": 113706.5, + "avg_turn_count": 14.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 42.531134366989136, + "agent_execution_time": 42.531134366989136, + "token_usage": { + "input_tokens": 101108, + "output_tokens": 4673, + "total_tokens": 105781, + "reasoning_tokens": 2128 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": true, + "execution_time": 38.95880889892578, + "agent_execution_time": 38.95880889892578, + "token_usage": { + "input_tokens": 73389, + "output_tokens": 2275, + "total_tokens": 75664, + "reasoning_tokens": 1028 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": true, + "execution_time": 46.56830334663391, + "agent_execution_time": 46.56830334663391, + "token_usage": { + "input_tokens": 77165, + "output_tokens": 3253, + "total_tokens": 80418, + "reasoning_tokens": 1903 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": true, + "execution_time": 38.77199149131775, + "agent_execution_time": 38.77199149131775, + "token_usage": { + "input_tokens": 71853, + "output_tokens": 3264, + "total_tokens": 75117, + "reasoning_tokens": 2311 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 41.71, + "avg_input_tokens": 80878.8, + "avg_output_tokens": 3366.2, + "avg_total_tokens": 84245.0, + "avg_turn_count": 13.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/employees__employee_project_tracking.json b/task_results/employees__employee_project_tracking.json index 4fb6b067..e7e243a1 100644 --- a/task_results/employees__employee_project_tracking.json +++ b/task_results/employees__employee_project_tracking.json @@ -2,6 +2,73 @@ "task": "employees__employee_project_tracking", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 272.468674659729, + "agent_execution_time": 272.468674659729, + "token_usage": { + "input_tokens": 251023, + "output_tokens": 2924, + "total_tokens": 253947, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-2", + "success": true, + "execution_time": 281.83456468582153, + "agent_execution_time": 281.83456468582153, + "token_usage": { + "input_tokens": 234144, + "output_tokens": 3163, + "total_tokens": 237307, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-3", + "success": false, + "execution_time": 278.30049180984497, + "agent_execution_time": 278.30049180984497, + "token_usage": { + "input_tokens": 109790, + "output_tokens": 985, + "total_tokens": 110775, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 291.89334869384766, + "agent_execution_time": 291.89334869384766, + "token_usage": { + "input_tokens": 21612, + "output_tokens": 7253, + "total_tokens": 28865, + "reasoning_tokens": 7064 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 281.12, + "avg_input_tokens": 154142.2, + "avg_output_tokens": 3581.2, + "avg_total_tokens": 157723.5, + "avg_turn_count": 21.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 450.58818912506104, - "agent_execution_time": 450.58818912506104, - "token_usage": { - "input_tokens": 213055, - "output_tokens": 7961, - "total_tokens": 221016, - "reasoning_tokens": null - }, - "turn_count": 30 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 450.59, - "avg_input_tokens": 213055.0, - "avg_output_tokens": 7961.0, - "avg_total_tokens": 221016.0, - "avg_turn_count": 30.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 195.40135407447815, + "agent_execution_time": 195.40135407447815, + "token_usage": { + "input_tokens": 690485, + "output_tokens": 7091, + "total_tokens": 697576, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + { + "run": "run-2", + "success": true, + "execution_time": 152.72725367546082, + "agent_execution_time": 152.72725367546082, + "token_usage": { + "input_tokens": 357027, + "output_tokens": 6806, + "total_tokens": 363833, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-3", + "success": true, + "execution_time": 165.87249898910522, + "agent_execution_time": 165.87249898910522, + "token_usage": { + "input_tokens": 377787, + "output_tokens": 7469, + "total_tokens": 385256, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": false, + "execution_time": 147.441415309906, + "agent_execution_time": 147.441415309906, + "token_usage": { + "input_tokens": 265250, + "output_tokens": 7180, + "total_tokens": 272430, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 165.36, + "avg_input_tokens": 422637.2, + "avg_output_tokens": 7136.5, + "avg_total_tokens": 429773.8, + "avg_turn_count": 29.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 310.671263217926, + "agent_execution_time": 310.671263217926, + "token_usage": { + "input_tokens": 380970, + "output_tokens": 5956, + "total_tokens": 386926, + "reasoning_tokens": 0 + }, + "turn_count": 41 + }, + { + "run": "run-2", + "success": true, + "execution_time": 242.3810329437256, + "agent_execution_time": 242.3810329437256, + "token_usage": { + "input_tokens": 392049, + "output_tokens": 5494, + "total_tokens": 397543, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + { + "run": "run-3", + "success": false, + "execution_time": 82.74441766738892, + "agent_execution_time": 82.74441766738892, + "token_usage": { + "input_tokens": 92605, + "output_tokens": 1495, + "total_tokens": 94100, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": true, + "execution_time": 224.98572301864624, + "agent_execution_time": 224.98572301864624, + "token_usage": { + "input_tokens": 433829, + "output_tokens": 6248, + "total_tokens": 440077, + "reasoning_tokens": 0 + }, + "turn_count": 44 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 215.2, + "avg_input_tokens": 324863.2, + "avg_output_tokens": 4798.2, + "avg_total_tokens": 329661.5, + "avg_turn_count": 34.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 254.79966402053833, + "agent_execution_time": 254.79966402053833, + "token_usage": { + "input_tokens": 279479, + "output_tokens": 3292, + "total_tokens": 282771, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-2", + "success": true, + "execution_time": 283.3107876777649, + "agent_execution_time": 283.3107876777649, + "token_usage": { + "input_tokens": 231103, + "output_tokens": 3711, + "total_tokens": 234814, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-3", + "success": true, + "execution_time": 244.51415610313416, + "agent_execution_time": 244.51415610313416, + "token_usage": { + "input_tokens": 233443, + "output_tokens": 2865, + "total_tokens": 236308, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.218586683273315, + "agent_execution_time": 8.218586683273315, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 197.71, + "avg_input_tokens": 186006.2, + "avg_output_tokens": 2467.0, + "avg_total_tokens": 188473.2, + "avg_turn_count": 23.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 38.21620297431946, + "agent_execution_time": 38.21620297431946, + "token_usage": { + "input_tokens": 105995, + "output_tokens": 4015, + "total_tokens": 110010, + "reasoning_tokens": 2170 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": true, + "execution_time": 61.35165214538574, + "agent_execution_time": 61.35165214538574, + "token_usage": { + "input_tokens": 189719, + "output_tokens": 5158, + "total_tokens": 194877, + "reasoning_tokens": 2441 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": true, + "execution_time": 50.22589993476868, + "agent_execution_time": 50.22589993476868, + "token_usage": { + "input_tokens": 126016, + "output_tokens": 4016, + "total_tokens": 130032, + "reasoning_tokens": 2054 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": true, + "execution_time": 49.095847606658936, + "agent_execution_time": 49.095847606658936, + "token_usage": { + "input_tokens": 137905, + "output_tokens": 4522, + "total_tokens": 142427, + "reasoning_tokens": 2495 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 49.72, + "avg_input_tokens": 139908.8, + "avg_output_tokens": 4427.8, + "avg_total_tokens": 144336.5, + "avg_turn_count": 18.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/employees__employee_retention_analysis.json b/task_results/employees__employee_retention_analysis.json index 63700e81..2aff52f0 100644 --- a/task_results/employees__employee_retention_analysis.json +++ b/task_results/employees__employee_retention_analysis.json @@ -2,6 +2,73 @@ "task": "employees__employee_retention_analysis", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 306.8211603164673, + "agent_execution_time": 306.8211603164673, + "token_usage": { + "input_tokens": 309162, + "output_tokens": 3581, + "total_tokens": 312743, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-2", + "success": false, + "execution_time": 322.8718523979187, + "agent_execution_time": 322.8718523979187, + "token_usage": { + "input_tokens": 284053, + "output_tokens": 5027, + "total_tokens": 289080, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-3", + "success": false, + "execution_time": 237.07192850112915, + "agent_execution_time": 237.07192850112915, + "token_usage": { + "input_tokens": 170293, + "output_tokens": 3422, + "total_tokens": 173715, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 105.83524990081787, + "agent_execution_time": 105.83524990081787, + "token_usage": { + "input_tokens": 19309, + "output_tokens": 2355, + "total_tokens": 21664, + "reasoning_tokens": 2187 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 243.15, + "avg_input_tokens": 195704.2, + "avg_output_tokens": 3596.2, + "avg_total_tokens": 199300.5, + "avg_turn_count": 21.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 758.7352375984192, - "agent_execution_time": 758.7352375984192, - "token_usage": { - "input_tokens": 375875, - "output_tokens": 15307, - "total_tokens": 391182, - "reasoning_tokens": null - }, - "turn_count": 31 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 758.74, - "avg_input_tokens": 375875.0, - "avg_output_tokens": 15307.0, - "avg_total_tokens": 391182.0, - "avg_turn_count": 31.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 145.07664322853088, + "agent_execution_time": 145.07664322853088, + "token_usage": { + "input_tokens": 323530, + "output_tokens": 6771, + "total_tokens": 330301, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": false, + "execution_time": 116.00009608268738, + "agent_execution_time": 116.00009608268738, + "token_usage": { + "input_tokens": 192720, + "output_tokens": 5286, + "total_tokens": 198006, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 170.12169933319092, + "agent_execution_time": 170.12169933319092, + "token_usage": { + "input_tokens": 499051, + "output_tokens": 7672, + "total_tokens": 506723, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": false, + "execution_time": 161.5325903892517, + "agent_execution_time": 161.5325903892517, + "token_usage": { + "input_tokens": 269092, + "output_tokens": 6949, + "total_tokens": 276041, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 148.18, + "avg_input_tokens": 321098.2, + "avg_output_tokens": 6669.5, + "avg_total_tokens": 327767.8, + "avg_turn_count": 19.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 201.9079568386078, + "agent_execution_time": 201.9079568386078, + "token_usage": { + "input_tokens": 159362, + "output_tokens": 6049, + "total_tokens": 165411, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 358.5479097366333, + "agent_execution_time": 358.5479097366333, + "token_usage": { + "input_tokens": 181584, + "output_tokens": 5361, + "total_tokens": 186945, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 10.116178035736084, + "agent_execution_time": 10.116178035736084, + "token_usage": { + "input_tokens": 7360, + "output_tokens": 200, + "total_tokens": 7560, + "reasoning_tokens": 0 + }, + "turn_count": 3 + }, + { + "run": "run-4", + "success": false, + "execution_time": 149.96325588226318, + "agent_execution_time": 149.96325588226318, + "token_usage": { + "input_tokens": 141993, + "output_tokens": 4819, + "total_tokens": 146812, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 180.13, + "avg_input_tokens": 122574.8, + "avg_output_tokens": 4107.2, + "avg_total_tokens": 126682.0, + "avg_turn_count": 15.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 349.061621427536, + "agent_execution_time": 349.061621427536, + "token_usage": { + "input_tokens": 290228, + "output_tokens": 5512, + "total_tokens": 295740, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-2", + "success": false, + "execution_time": 264.4508104324341, + "agent_execution_time": 264.4508104324341, + "token_usage": { + "input_tokens": 183875, + "output_tokens": 3991, + "total_tokens": 187866, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 298.1451494693756, + "agent_execution_time": 298.1451494693756, + "token_usage": { + "input_tokens": 234176, + "output_tokens": 4348, + "total_tokens": 238524, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.207370042800903, + "agent_execution_time": 8.207370042800903, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 229.97, + "avg_input_tokens": 177069.8, + "avg_output_tokens": 3462.8, + "avg_total_tokens": 180532.5, + "avg_turn_count": 19.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 112.12750005722046, + "agent_execution_time": 112.12750005722046, + "token_usage": { + "input_tokens": 343929, + "output_tokens": 9799, + "total_tokens": 353728, + "reasoning_tokens": 6780 + }, + "turn_count": 30 + }, + { + "run": "run-2", + "success": true, + "execution_time": 71.78340077400208, + "agent_execution_time": 71.78340077400208, + "token_usage": { + "input_tokens": 134433, + "output_tokens": 7488, + "total_tokens": 141921, + "reasoning_tokens": 5658 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": true, + "execution_time": 45.265514612197876, + "agent_execution_time": 45.265514612197876, + "token_usage": { + "input_tokens": 52759, + "output_tokens": 5291, + "total_tokens": 58050, + "reasoning_tokens": 3772 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 67.05810022354126, + "agent_execution_time": 67.05810022354126, + "token_usage": { + "input_tokens": 109790, + "output_tokens": 5892, + "total_tokens": 115682, + "reasoning_tokens": 4339 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 74.06, + "avg_input_tokens": 160227.8, + "avg_output_tokens": 7117.5, + "avg_total_tokens": 167345.2, + "avg_turn_count": 17.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/employees__executive_dashboard_automation.json b/task_results/employees__executive_dashboard_automation.json index 324e5ed6..04794fad 100644 --- a/task_results/employees__executive_dashboard_automation.json +++ b/task_results/employees__executive_dashboard_automation.json @@ -2,6 +2,73 @@ "task": "employees__executive_dashboard_automation", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 307.86621141433716, + "agent_execution_time": 307.86621141433716, + "token_usage": { + "input_tokens": 275964, + "output_tokens": 4327, + "total_tokens": 280291, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 305.2025291919708, + "agent_execution_time": 305.2025291919708, + "token_usage": { + "input_tokens": 311463, + "output_tokens": 4370, + "total_tokens": 315833, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1015.3421957492828, + "agent_execution_time": 1015.3421957492828, + "token_usage": { + "input_tokens": 209079, + "output_tokens": 27140, + "total_tokens": 236219, + "reasoning_tokens": 23493 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 640.9347364902496, + "agent_execution_time": 640.9347364902496, + "token_usage": { + "input_tokens": 21215, + "output_tokens": 16921, + "total_tokens": 38136, + "reasoning_tokens": 13951 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 567.34, + "avg_input_tokens": 204430.2, + "avg_output_tokens": 13189.5, + "avg_total_tokens": 217619.8, + "avg_turn_count": 19.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 552.2760052680969, - "agent_execution_time": 552.2760052680969, - "token_usage": { - "input_tokens": 304039, - "output_tokens": 10634, - "total_tokens": 314673, - "reasoning_tokens": null - }, - "turn_count": 31 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 552.28, - "avg_input_tokens": 304039.0, - "avg_output_tokens": 10634.0, - "avg_total_tokens": 314673.0, - "avg_turn_count": 31.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 183.95886039733887, + "agent_execution_time": 183.95886039733887, + "token_usage": { + "input_tokens": 532213, + "output_tokens": 6843, + "total_tokens": 539056, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-2", + "success": false, + "execution_time": 189.57184600830078, + "agent_execution_time": 189.57184600830078, + "token_usage": { + "input_tokens": 593971, + "output_tokens": 7876, + "total_tokens": 601847, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + { + "run": "run-3", + "success": false, + "execution_time": 341.2478756904602, + "agent_execution_time": 341.2478756904602, + "token_usage": { + "input_tokens": 671135, + "output_tokens": 12626, + "total_tokens": 683761, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + { + "run": "run-4", + "success": false, + "execution_time": 230.7133400440216, + "agent_execution_time": 230.7133400440216, + "token_usage": { + "input_tokens": 492559, + "output_tokens": 8415, + "total_tokens": 500974, + "reasoning_tokens": 0 + }, + "turn_count": 32 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 236.37, + "avg_input_tokens": 572469.5, + "avg_output_tokens": 8940.0, + "avg_total_tokens": 581409.5, + "avg_turn_count": 34.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 182.29265069961548, + "agent_execution_time": 182.29265069961548, + "token_usage": { + "input_tokens": 376611, + "output_tokens": 4875, + "total_tokens": 381486, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + { + "run": "run-2", + "success": false, + "execution_time": 380.42989230155945, + "agent_execution_time": 380.42989230155945, + "token_usage": { + "input_tokens": 271454, + "output_tokens": 3455, + "total_tokens": 274909, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-3", + "success": false, + "execution_time": 233.18688774108887, + "agent_execution_time": 233.18688774108887, + "token_usage": { + "input_tokens": 321311, + "output_tokens": 5571, + "total_tokens": 326882, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + { + "run": "run-4", + "success": false, + "execution_time": 194.60288000106812, + "agent_execution_time": 194.60288000106812, + "token_usage": { + "input_tokens": 282773, + "output_tokens": 5174, + "total_tokens": 287947, + "reasoning_tokens": 0 + }, + "turn_count": 31 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 247.63, + "avg_input_tokens": 313037.2, + "avg_output_tokens": 4768.8, + "avg_total_tokens": 317806.0, + "avg_turn_count": 32.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 390.2959725856781, + "agent_execution_time": 390.2959725856781, + "token_usage": { + "input_tokens": 438405, + "output_tokens": 5526, + "total_tokens": 443931, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + { + "run": "run-2", + "success": false, + "execution_time": 359.1146535873413, + "agent_execution_time": 359.1146535873413, + "token_usage": { + "input_tokens": 300441, + "output_tokens": 4498, + "total_tokens": 304939, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-3", + "success": false, + "execution_time": 283.06897473335266, + "agent_execution_time": 283.06897473335266, + "token_usage": { + "input_tokens": 285543, + "output_tokens": 3465, + "total_tokens": 289008, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.23619031906128, + "agent_execution_time": 8.23619031906128, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 260.18, + "avg_input_tokens": 256097.2, + "avg_output_tokens": 3372.2, + "avg_total_tokens": 259469.5, + "avg_turn_count": 24.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 112.6349561214447, + "agent_execution_time": 112.6349561214447, + "token_usage": { + "input_tokens": 515137, + "output_tokens": 10512, + "total_tokens": 525649, + "reasoning_tokens": 6654 + }, + "turn_count": 39 + }, + { + "run": "run-2", + "success": false, + "execution_time": 74.55596232414246, + "agent_execution_time": 74.55596232414246, + "token_usage": { + "input_tokens": 292539, + "output_tokens": 5974, + "total_tokens": 298513, + "reasoning_tokens": 3233 + }, + "turn_count": 28 + }, + { + "run": "run-3", + "success": false, + "execution_time": 129.5191683769226, + "agent_execution_time": 129.5191683769226, + "token_usage": { + "input_tokens": 288811, + "output_tokens": 8704, + "total_tokens": 297515, + "reasoning_tokens": 5313 + }, + "turn_count": 25 + }, + { + "run": "run-4", + "success": false, + "execution_time": 141.3994083404541, + "agent_execution_time": 141.3994083404541, + "token_usage": { + "input_tokens": 658363, + "output_tokens": 12204, + "total_tokens": 670567, + "reasoning_tokens": 7619 + }, + "turn_count": 41 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 114.53, + "avg_input_tokens": 438712.5, + "avg_output_tokens": 9348.5, + "avg_total_tokens": 448061.0, + "avg_turn_count": 33.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/employees__management_structure_analysis.json b/task_results/employees__management_structure_analysis.json index 711a5b7b..3972b1cb 100644 --- a/task_results/employees__management_structure_analysis.json +++ b/task_results/employees__management_structure_analysis.json @@ -2,6 +2,73 @@ "task": "employees__management_structure_analysis", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 237.82022404670715, + "agent_execution_time": 237.82022404670715, + "token_usage": { + "input_tokens": 227506, + "output_tokens": 3184, + "total_tokens": 230690, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 249.29724287986755, + "agent_execution_time": 249.29724287986755, + "token_usage": { + "input_tokens": 195473, + "output_tokens": 3643, + "total_tokens": 199116, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": true, + "execution_time": 795.6358561515808, + "agent_execution_time": 795.6358561515808, + "token_usage": { + "input_tokens": 61521, + "output_tokens": 21387, + "total_tokens": 82908, + "reasoning_tokens": 18882 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": true, + "execution_time": 438.61567544937134, + "agent_execution_time": 438.61567544937134, + "token_usage": { + "input_tokens": 38820, + "output_tokens": 10377, + "total_tokens": 49197, + "reasoning_tokens": 8762 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 430.34, + "avg_input_tokens": 130830.0, + "avg_output_tokens": 9647.8, + "avg_total_tokens": 140477.8, + "avg_turn_count": 15.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 534.903430223465, - "agent_execution_time": 534.903430223465, - "token_usage": { - "input_tokens": 273476, - "output_tokens": 10140, - "total_tokens": 283616, - "reasoning_tokens": null - }, - "turn_count": 28 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 534.9, - "avg_input_tokens": 273476.0, - "avg_output_tokens": 10140.0, - "avg_total_tokens": 283616.0, - "avg_turn_count": 28.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 312.3840157985687, + "agent_execution_time": 312.3840157985687, + "token_usage": { + "input_tokens": 292459, + "output_tokens": 6350, + "total_tokens": 298809, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": true, + "execution_time": 148.8743360042572, + "agent_execution_time": 148.8743360042572, + "token_usage": { + "input_tokens": 360174, + "output_tokens": 7572, + "total_tokens": 367746, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": true, + "execution_time": 170.11345911026, + "agent_execution_time": 170.11345911026, + "token_usage": { + "input_tokens": 446019, + "output_tokens": 8377, + "total_tokens": 454396, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-4", + "success": true, + "execution_time": 118.14211320877075, + "agent_execution_time": 118.14211320877075, + "token_usage": { + "input_tokens": 240100, + "output_tokens": 5535, + "total_tokens": 245635, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 187.38, + "avg_input_tokens": 334688.0, + "avg_output_tokens": 6958.5, + "avg_total_tokens": 341646.5, + "avg_turn_count": 20.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 185.06957983970642, + "agent_execution_time": 185.06957983970642, + "token_usage": { + "input_tokens": 259300, + "output_tokens": 5972, + "total_tokens": 265272, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": true, + "execution_time": 522.5048370361328, + "agent_execution_time": 522.5048370361328, + "token_usage": { + "input_tokens": 320170, + "output_tokens": 5118, + "total_tokens": 325288, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-3", + "success": true, + "execution_time": 397.9401066303253, + "agent_execution_time": 397.9401066303253, + "token_usage": { + "input_tokens": 344172, + "output_tokens": 5855, + "total_tokens": 350027, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-4", + "success": true, + "execution_time": 151.05041193962097, + "agent_execution_time": 151.05041193962097, + "token_usage": { + "input_tokens": 212779, + "output_tokens": 4759, + "total_tokens": 217538, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 314.14, + "avg_input_tokens": 284105.2, + "avg_output_tokens": 5426.0, + "avg_total_tokens": 289531.2, + "avg_turn_count": 28.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 225.8659896850586, + "agent_execution_time": 225.8659896850586, + "token_usage": { + "input_tokens": 235075, + "output_tokens": 3420, + "total_tokens": 238495, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": true, + "execution_time": 189.64699697494507, + "agent_execution_time": 189.64699697494507, + "token_usage": { + "input_tokens": 165854, + "output_tokens": 2568, + "total_tokens": 168422, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": false, + "execution_time": 208.36173677444458, + "agent_execution_time": 208.36173677444458, + "token_usage": { + "input_tokens": 185316, + "output_tokens": 2734, + "total_tokens": 188050, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.230656862258911, + "agent_execution_time": 8.230656862258911, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 158.03, + "avg_input_tokens": 146561.2, + "avg_output_tokens": 2180.5, + "avg_total_tokens": 148741.8, + "avg_turn_count": 16.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 43.32529592514038, + "agent_execution_time": 43.32529592514038, + "token_usage": { + "input_tokens": 100035, + "output_tokens": 5093, + "total_tokens": 105128, + "reasoning_tokens": 3571 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": true, + "execution_time": 137.73046112060547, + "agent_execution_time": 137.73046112060547, + "token_usage": { + "input_tokens": 483145, + "output_tokens": 10056, + "total_tokens": 493201, + "reasoning_tokens": 5341 + }, + "turn_count": 39 + }, + { + "run": "run-3", + "success": true, + "execution_time": 67.06036972999573, + "agent_execution_time": 67.06036972999573, + "token_usage": { + "input_tokens": 174427, + "output_tokens": 5259, + "total_tokens": 179686, + "reasoning_tokens": 3405 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": true, + "execution_time": 67.87331104278564, + "agent_execution_time": 67.87331104278564, + "token_usage": { + "input_tokens": 134790, + "output_tokens": 7062, + "total_tokens": 141852, + "reasoning_tokens": 5300 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 79.0, + "avg_input_tokens": 223099.2, + "avg_output_tokens": 6867.5, + "avg_total_tokens": 229966.8, + "avg_turn_count": 23.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/eval_web__cloudflare_turnstile_challenge.json b/task_results/eval_web__cloudflare_turnstile_challenge.json index 3a72fb49..55aff012 100644 --- a/task_results/eval_web__cloudflare_turnstile_challenge.json +++ b/task_results/eval_web__cloudflare_turnstile_challenge.json @@ -2,6 +2,73 @@ "task": "eval_web__cloudflare_turnstile_challenge", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 569.7963891029358, + "agent_execution_time": 569.7963891029358, + "token_usage": { + "input_tokens": 996745, + "output_tokens": 8078, + "total_tokens": 1004823, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + { + "run": "run-2", + "success": false, + "execution_time": 505.6805808544159, + "agent_execution_time": 505.6805808544159, + "token_usage": { + "input_tokens": 655707, + "output_tokens": 8209, + "total_tokens": 663916, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-3", + "success": false, + "execution_time": 409.2838089466095, + "agent_execution_time": 409.2838089466095, + "token_usage": { + "input_tokens": 258539, + "output_tokens": 7988, + "total_tokens": 266527, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 424.75878071784973, + "agent_execution_time": 424.75878071784973, + "token_usage": { + "input_tokens": 351222, + "output_tokens": 6008, + "total_tokens": 357230, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 477.38, + "avg_input_tokens": 565553.2, + "avg_output_tokens": 7570.8, + "avg_total_tokens": 573124.0, + "avg_turn_count": 28.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 565.0432152748108, - "agent_execution_time": 565.0432152748108, - "token_usage": { - "input_tokens": 548500, - "output_tokens": 7703, - "total_tokens": 556203, - "reasoning_tokens": null - }, - "turn_count": 25 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 565.04, - "avg_input_tokens": 548500.0, - "avg_output_tokens": 7703.0, - "avg_total_tokens": 556203.0, - "avg_turn_count": 25.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 253.90698075294495, + "agent_execution_time": 253.90698075294495, + "token_usage": { + "input_tokens": 1753819, + "output_tokens": 4654, + "total_tokens": 1758473, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 570.5327370166779, + "agent_execution_time": 570.5327370166779, + "token_usage": { + "input_tokens": 2558253, + "output_tokens": 15098, + "total_tokens": 2573351, + "reasoning_tokens": 0 + }, + "turn_count": 64 + }, + { + "run": "run-3", + "success": false, + "execution_time": 273.87959909439087, + "agent_execution_time": 273.87959909439087, + "token_usage": { + "input_tokens": 1868567, + "output_tokens": 7582, + "total_tokens": 1876149, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + { + "run": "run-4", + "success": false, + "execution_time": 323.2769525051117, + "agent_execution_time": 323.2769525051117, + "token_usage": { + "input_tokens": 1430075, + "output_tokens": 8372, + "total_tokens": 1438447, + "reasoning_tokens": 0 + }, + "turn_count": 47 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 355.4, + "avg_input_tokens": 1902678.5, + "avg_output_tokens": 8926.5, + "avg_total_tokens": 1911605.0, + "avg_turn_count": 40.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 7.055896759033203, + "agent_execution_time": 7.055896759033203, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-2", + "success": false, + "execution_time": 377.9859049320221, + "agent_execution_time": 377.9859049320221, + "token_usage": { + "input_tokens": 688625, + "output_tokens": 9010, + "total_tokens": 697635, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-3", + "success": false, + "execution_time": 343.5885910987854, + "agent_execution_time": 343.5885910987854, + "token_usage": { + "input_tokens": 665935, + "output_tokens": 9295, + "total_tokens": 675230, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-4", + "success": false, + "execution_time": 142.8690106868744, + "agent_execution_time": 142.8690106868744, + "token_usage": { + "input_tokens": 169294, + "output_tokens": 4404, + "total_tokens": 173698, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 217.87, + "avg_input_tokens": 380963.5, + "avg_output_tokens": 5677.2, + "avg_total_tokens": 386640.8, + "avg_turn_count": 20.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 340.6907594203949, + "agent_execution_time": 340.6907594203949, + "token_usage": { + "input_tokens": 343102, + "output_tokens": 5100, + "total_tokens": 348202, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": false, + "execution_time": 355.706668138504, + "agent_execution_time": 355.706668138504, + "token_usage": { + "input_tokens": 294153, + "output_tokens": 5915, + "total_tokens": 300068, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 318.0956618785858, + "agent_execution_time": 318.0956618785858, + "token_usage": { + "input_tokens": 301839, + "output_tokens": 5175, + "total_tokens": 307014, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 387.7790365219116, + "agent_execution_time": 387.7790365219116, + "token_usage": { + "input_tokens": 403258, + "output_tokens": 4783, + "total_tokens": 408041, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 350.57, + "avg_input_tokens": 335588.0, + "avg_output_tokens": 5243.2, + "avg_total_tokens": 340831.2, + "avg_turn_count": 22.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 145.92757034301758, + "agent_execution_time": 145.92757034301758, + "token_usage": { + "input_tokens": 1779538, + "output_tokens": 5420, + "total_tokens": 1784958, + "reasoning_tokens": 4164 + }, + "turn_count": 34 + }, + { + "run": "run-2", + "success": false, + "execution_time": 74.85641479492188, + "agent_execution_time": 74.85641479492188, + "token_usage": { + "input_tokens": 807838, + "output_tokens": 3620, + "total_tokens": 811458, + "reasoning_tokens": 2829 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 160.32092142105103, + "agent_execution_time": 160.32092142105103, + "token_usage": { + "input_tokens": 1726538, + "output_tokens": 6651, + "total_tokens": 1733189, + "reasoning_tokens": 5075 + }, + "turn_count": 32 + }, + { + "run": "run-4", + "success": false, + "execution_time": 233.87894129753113, + "agent_execution_time": 233.87894129753113, + "token_usage": { + "input_tokens": 2623119, + "output_tokens": 11261, + "total_tokens": 2634380, + "reasoning_tokens": 7880 + }, + "turn_count": 50 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 153.75, + "avg_input_tokens": 1734258.2, + "avg_output_tokens": 6738.0, + "avg_total_tokens": 1740996.2, + "avg_turn_count": 33.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/eval_web__extraction_table.json b/task_results/eval_web__extraction_table.json index 2ce08d23..88ba61a6 100644 --- a/task_results/eval_web__extraction_table.json +++ b/task_results/eval_web__extraction_table.json @@ -2,6 +2,73 @@ "task": "eval_web__extraction_table", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 428.7061369419098, + "agent_execution_time": 428.7061369419098, + "token_usage": { + "input_tokens": 1009113, + "output_tokens": 6129, + "total_tokens": 1015242, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-2", + "success": true, + "execution_time": 325.54053139686584, + "agent_execution_time": 325.54053139686584, + "token_usage": { + "input_tokens": 701525, + "output_tokens": 5203, + "total_tokens": 706728, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": false, + "execution_time": 352.2859034538269, + "agent_execution_time": 352.2859034538269, + "token_usage": { + "input_tokens": 737270, + "output_tokens": 5313, + "total_tokens": 742583, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 423.85006046295166, + "agent_execution_time": 423.85006046295166, + "token_usage": { + "input_tokens": 714910, + "output_tokens": 5815, + "total_tokens": 720725, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 382.6, + "avg_input_tokens": 790704.5, + "avg_output_tokens": 5615.0, + "avg_total_tokens": 796319.5, + "avg_turn_count": 24.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 151.304541349411, - "agent_execution_time": 151.304541349411, - "token_usage": { - "input_tokens": 33113, - "output_tokens": 2322, - "total_tokens": 35435, - "reasoning_tokens": null - }, - "turn_count": 4 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 151.3, - "avg_input_tokens": 33113.0, - "avg_output_tokens": 2322.0, - "avg_total_tokens": 35435.0, - "avg_turn_count": 4.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 80.66271924972534, + "agent_execution_time": 80.66271924972534, + "token_usage": { + "input_tokens": 108190, + "output_tokens": 3274, + "total_tokens": 111464, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 72.62936973571777, + "agent_execution_time": 72.62936973571777, + "token_usage": { + "input_tokens": 100572, + "output_tokens": 3034, + "total_tokens": 103606, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 52.02479100227356, + "agent_execution_time": 52.02479100227356, + "token_usage": { + "input_tokens": 24904, + "output_tokens": 2321, + "total_tokens": 27225, + "reasoning_tokens": 0 + }, + "turn_count": 3 + }, + { + "run": "run-4", + "success": false, + "execution_time": 79.66497540473938, + "agent_execution_time": 79.66497540473938, + "token_usage": { + "input_tokens": 111918, + "output_tokens": 3491, + "total_tokens": 115409, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 71.25, + "avg_input_tokens": 86396.0, + "avg_output_tokens": 3030.0, + "avg_total_tokens": 89426.0, + "avg_turn_count": 6.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 307.0022292137146, + "agent_execution_time": 307.0022292137146, + "token_usage": { + "input_tokens": 628481, + "output_tokens": 6534, + "total_tokens": 635015, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": true, + "execution_time": 319.55122542381287, + "agent_execution_time": 319.55122542381287, + "token_usage": { + "input_tokens": 644006, + "output_tokens": 8224, + "total_tokens": 652230, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": true, + "execution_time": 227.08725810050964, + "agent_execution_time": 227.08725810050964, + "token_usage": { + "input_tokens": 194261, + "output_tokens": 5557, + "total_tokens": 199818, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 163.77771735191345, + "agent_execution_time": 163.77771735191345, + "token_usage": { + "input_tokens": 66785, + "output_tokens": 6079, + "total_tokens": 72864, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 254.35, + "avg_input_tokens": 383383.2, + "avg_output_tokens": 6598.5, + "avg_total_tokens": 389981.8, + "avg_turn_count": 15.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 141.41911959648132, + "agent_execution_time": 141.41911959648132, + "token_usage": { + "input_tokens": 150154, + "output_tokens": 2047, + "total_tokens": 152201, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 506.0817856788635, + "agent_execution_time": 506.0817856788635, + "token_usage": { + "input_tokens": 1369313, + "output_tokens": 7514, + "total_tokens": 1376827, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-3", + "success": true, + "execution_time": 311.383891582489, + "agent_execution_time": 311.383891582489, + "token_usage": { + "input_tokens": 677156, + "output_tokens": 4514, + "total_tokens": 681670, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": true, + "execution_time": 367.910715341568, + "agent_execution_time": 367.910715341568, + "token_usage": { + "input_tokens": 677234, + "output_tokens": 4859, + "total_tokens": 682093, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 331.7, + "avg_input_tokens": 718464.2, + "avg_output_tokens": 4733.5, + "avg_total_tokens": 723197.8, + "avg_turn_count": 21.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 75.35624742507935, + "agent_execution_time": 75.35624742507935, + "token_usage": { + "input_tokens": 102036, + "output_tokens": 5233, + "total_tokens": 107269, + "reasoning_tokens": 2523 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 61.36948490142822, + "agent_execution_time": 61.36948490142822, + "token_usage": { + "input_tokens": 76636, + "output_tokens": 5562, + "total_tokens": 82198, + "reasoning_tokens": 3888 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 25.2391197681427, + "agent_execution_time": 25.2391197681427, + "token_usage": { + "input_tokens": 17839, + "output_tokens": 2672, + "total_tokens": 20511, + "reasoning_tokens": 2265 + }, + "turn_count": 3 + }, + { + "run": "run-4", + "success": false, + "execution_time": 58.82275152206421, + "agent_execution_time": 58.82275152206421, + "token_usage": { + "input_tokens": 19945, + "output_tokens": 5074, + "total_tokens": 25019, + "reasoning_tokens": 3017 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 55.2, + "avg_input_tokens": 54114.0, + "avg_output_tokens": 4635.2, + "avg_total_tokens": 58749.2, + "avg_turn_count": 5.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/file_context__duplicates_searching.json b/task_results/file_context__duplicates_searching.json index c51cc736..a55aed88 100644 --- a/task_results/file_context__duplicates_searching.json +++ b/task_results/file_context__duplicates_searching.json @@ -2,6 +2,73 @@ "task": "file_context__duplicates_searching", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 153.30653977394104, + "agent_execution_time": 153.30653977394104, + "token_usage": { + "input_tokens": 82054, + "output_tokens": 2291, + "total_tokens": 84345, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": true, + "execution_time": 624.7218894958496, + "agent_execution_time": 624.7218894958496, + "token_usage": { + "input_tokens": 39607, + "output_tokens": 17869, + "total_tokens": 57476, + "reasoning_tokens": 15423 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": true, + "execution_time": 643.4807195663452, + "agent_execution_time": 643.4807195663452, + "token_usage": { + "input_tokens": 33792, + "output_tokens": 16341, + "total_tokens": 50133, + "reasoning_tokens": 13667 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 852.3943758010864, + "agent_execution_time": 852.3943758010864, + "token_usage": { + "input_tokens": 32832, + "output_tokens": 17130, + "total_tokens": 49962, + "reasoning_tokens": 15242 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 568.48, + "avg_input_tokens": 47071.2, + "avg_output_tokens": 13407.8, + "avg_total_tokens": 60479.0, + "avg_turn_count": 8.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 189.84558725357056, - "agent_execution_time": 189.84558725357056, - "token_usage": { - "input_tokens": 87748, - "output_tokens": 3673, - "total_tokens": 91421, - "reasoning_tokens": null - }, - "turn_count": 14 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 189.85, - "avg_input_tokens": 87748.0, - "avg_output_tokens": 3673.0, - "avg_total_tokens": 91421.0, - "avg_turn_count": 14.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 89.57424807548523, + "agent_execution_time": 89.57424807548523, + "token_usage": { + "input_tokens": 44023, + "output_tokens": 3856, + "total_tokens": 47879, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 54.63117861747742, + "agent_execution_time": 54.63117861747742, + "token_usage": { + "input_tokens": 44273, + "output_tokens": 3853, + "total_tokens": 48126, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 44.39495873451233, + "agent_execution_time": 44.39495873451233, + "token_usage": { + "input_tokens": 40743, + "output_tokens": 3046, + "total_tokens": 43789, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": true, + "execution_time": 53.31755542755127, + "agent_execution_time": 53.31755542755127, + "token_usage": { + "input_tokens": 49810, + "output_tokens": 3916, + "total_tokens": 53726, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 60.48, + "avg_input_tokens": 44712.2, + "avg_output_tokens": 3667.8, + "avg_total_tokens": 48380.0, + "avg_turn_count": 7.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 61.74481463432312, + "agent_execution_time": 61.74481463432312, + "token_usage": { + "input_tokens": 70967, + "output_tokens": 2367, + "total_tokens": 73334, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 43.0725634098053, + "agent_execution_time": 43.0725634098053, + "token_usage": { + "input_tokens": 69579, + "output_tokens": 2440, + "total_tokens": 72019, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 61.74481463432312, + "agent_execution_time": 61.74481463432312, + "token_usage": { + "input_tokens": 70967, + "output_tokens": 2367, + "total_tokens": 73334, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 43.0725634098053, + "agent_execution_time": 43.0725634098053, + "token_usage": { + "input_tokens": 69579, + "output_tokens": 2440, + "total_tokens": 72019, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 52.41, + "avg_input_tokens": 70273.0, + "avg_output_tokens": 2403.5, + "avg_total_tokens": 72676.5, + "avg_turn_count": 13.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 156.33373355865479, + "agent_execution_time": 156.33373355865479, + "token_usage": { + "input_tokens": 82328, + "output_tokens": 2312, + "total_tokens": 84640, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 148.13799571990967, + "agent_execution_time": 148.13799571990967, + "token_usage": { + "input_tokens": 73977, + "output_tokens": 2198, + "total_tokens": 76175, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 13.01108980178833, + "agent_execution_time": 13.01108980178833, + "token_usage": { + "input_tokens": 3033, + "output_tokens": 172, + "total_tokens": 3205, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 97.14682984352112, + "agent_execution_time": 97.14682984352112, + "token_usage": { + "input_tokens": 34981, + "output_tokens": 2098, + "total_tokens": 37079, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 103.66, + "avg_input_tokens": 48579.8, + "avg_output_tokens": 1695.0, + "avg_total_tokens": 50274.8, + "avg_turn_count": 9.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 31.07046866416931, + "agent_execution_time": 31.07046866416931, + "token_usage": { + "input_tokens": 26848, + "output_tokens": 3670, + "total_tokens": 30518, + "reasoning_tokens": 1493 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": true, + "execution_time": 26.182461500167847, + "agent_execution_time": 26.182461500167847, + "token_usage": { + "input_tokens": 27086, + "output_tokens": 3681, + "total_tokens": 30767, + "reasoning_tokens": 1504 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": true, + "execution_time": 19.116021394729614, + "agent_execution_time": 19.116021394729614, + "token_usage": { + "input_tokens": 26537, + "output_tokens": 3409, + "total_tokens": 29946, + "reasoning_tokens": 1232 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": true, + "execution_time": 24.408891916275024, + "agent_execution_time": 24.408891916275024, + "token_usage": { + "input_tokens": 26791, + "output_tokens": 3480, + "total_tokens": 30271, + "reasoning_tokens": 1303 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 25.19, + "avg_input_tokens": 26815.5, + "avg_output_tokens": 3560.0, + "avg_total_tokens": 30375.5, + "avg_turn_count": 6.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/file_context__file_merging.json b/task_results/file_context__file_merging.json index 68770633..e944c0c9 100644 --- a/task_results/file_context__file_merging.json +++ b/task_results/file_context__file_merging.json @@ -2,6 +2,73 @@ "task": "file_context__file_merging", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 78.707355260849, + "agent_execution_time": 78.707355260849, + "token_usage": { + "input_tokens": 20206, + "output_tokens": 1340, + "total_tokens": 21546, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": true, + "execution_time": 559.9810602664948, + "agent_execution_time": 559.9810602664948, + "token_usage": { + "input_tokens": 23358, + "output_tokens": 15322, + "total_tokens": 38680, + "reasoning_tokens": 14273 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": true, + "execution_time": 531.8824737071991, + "agent_execution_time": 531.8824737071991, + "token_usage": { + "input_tokens": 19174, + "output_tokens": 14058, + "total_tokens": 33232, + "reasoning_tokens": 13106 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": true, + "execution_time": 764.0157697200775, + "agent_execution_time": 764.0157697200775, + "token_usage": { + "input_tokens": 27109, + "output_tokens": 19977, + "total_tokens": 47086, + "reasoning_tokens": 18842 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 483.65, + "avg_input_tokens": 22461.8, + "avg_output_tokens": 12674.2, + "avg_total_tokens": 35136.0, + "avg_turn_count": 6.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 91.51880502700806, - "agent_execution_time": 91.51880502700806, - "token_usage": { - "input_tokens": 16470, - "output_tokens": 1823, - "total_tokens": 18293, - "reasoning_tokens": null - }, - "turn_count": 5 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 91.52, - "avg_input_tokens": 16470.0, - "avg_output_tokens": 1823.0, - "avg_total_tokens": 18293.0, - "avg_turn_count": 5.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 48.37791299819946, + "agent_execution_time": 48.37791299819946, + "token_usage": { + "input_tokens": 27646, + "output_tokens": 1918, + "total_tokens": 29564, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 36.249232053756714, + "agent_execution_time": 36.249232053756714, + "token_usage": { + "input_tokens": 28563, + "output_tokens": 1909, + "total_tokens": 30472, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 35.9812114238739, + "agent_execution_time": 35.9812114238739, + "token_usage": { + "input_tokens": 30825, + "output_tokens": 1838, + "total_tokens": 32663, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": true, + "execution_time": 36.04903721809387, + "agent_execution_time": 36.04903721809387, + "token_usage": { + "input_tokens": 28391, + "output_tokens": 1944, + "total_tokens": 30335, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 39.16, + "avg_input_tokens": 28856.2, + "avg_output_tokens": 1902.2, + "avg_total_tokens": 30758.5, + "avg_turn_count": 7.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 26.982781171798706, + "agent_execution_time": 26.982781171798706, + "token_usage": { + "input_tokens": 13787, + "output_tokens": 1318, + "total_tokens": 15105, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": true, + "execution_time": 13.764727592468262, + "agent_execution_time": 13.764727592468262, + "token_usage": { + "input_tokens": 13840, + "output_tokens": 1254, + "total_tokens": 15094, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 26.982781171798706, + "agent_execution_time": 26.982781171798706, + "token_usage": { + "input_tokens": 13787, + "output_tokens": 1318, + "total_tokens": 15105, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": true, + "execution_time": 13.764727592468262, + "agent_execution_time": 13.764727592468262, + "token_usage": { + "input_tokens": 13840, + "output_tokens": 1254, + "total_tokens": 15094, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 20.37, + "avg_input_tokens": 13813.5, + "avg_output_tokens": 1286.0, + "avg_total_tokens": 15099.5, + "avg_turn_count": 5.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 82.26133799552917, + "agent_execution_time": 82.26133799552917, + "token_usage": { + "input_tokens": 23533, + "output_tokens": 1363, + "total_tokens": 24896, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 161.9977617263794, + "agent_execution_time": 161.9977617263794, + "token_usage": { + "input_tokens": 60378, + "output_tokens": 2733, + "total_tokens": 63111, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 121.05527997016907, + "agent_execution_time": 121.05527997016907, + "token_usage": { + "input_tokens": 37450, + "output_tokens": 2511, + "total_tokens": 39961, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 59.1451141834259, + "agent_execution_time": 59.1451141834259, + "token_usage": { + "input_tokens": 19170, + "output_tokens": 1064, + "total_tokens": 20234, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 106.11, + "avg_input_tokens": 35132.8, + "avg_output_tokens": 1917.8, + "avg_total_tokens": 37050.5, + "avg_turn_count": 7.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 28.2689528465271, + "agent_execution_time": 28.2689528465271, + "token_usage": { + "input_tokens": 20372, + "output_tokens": 2497, + "total_tokens": 22869, + "reasoning_tokens": 1649 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 16.43811583518982, + "agent_execution_time": 16.43811583518982, + "token_usage": { + "input_tokens": 20052, + "output_tokens": 2070, + "total_tokens": 22122, + "reasoning_tokens": 1196 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 20.4749493598938, + "agent_execution_time": 20.4749493598938, + "token_usage": { + "input_tokens": 16853, + "output_tokens": 2240, + "total_tokens": 19093, + "reasoning_tokens": 1416 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": true, + "execution_time": 15.826157569885254, + "agent_execution_time": 15.826157569885254, + "token_usage": { + "input_tokens": 16831, + "output_tokens": 2207, + "total_tokens": 19038, + "reasoning_tokens": 1462 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 20.25, + "avg_input_tokens": 18527.0, + "avg_output_tokens": 2253.5, + "avg_total_tokens": 20780.5, + "avg_turn_count": 5.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/file_context__file_splitting.json b/task_results/file_context__file_splitting.json index 76183175..5660bef9 100644 --- a/task_results/file_context__file_splitting.json +++ b/task_results/file_context__file_splitting.json @@ -2,6 +2,73 @@ "task": "file_context__file_splitting", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 52.51876521110535, + "agent_execution_time": 52.51876521110535, + "token_usage": { + "input_tokens": 16014, + "output_tokens": 623, + "total_tokens": 16637, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 2419.200646162033, + "agent_execution_time": 2419.200646162033, + "token_usage": { + "input_tokens": 32580, + "output_tokens": 33562, + "total_tokens": 66142, + "reasoning_tokens": 32353 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 2839.5280299186707, + "agent_execution_time": 2839.5280299186707, + "token_usage": { + "input_tokens": 17345, + "output_tokens": 28203, + "total_tokens": 45548, + "reasoning_tokens": 25739 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 3600.1284716129303, + "agent_execution_time": 3600.1284716129303, + "token_usage": { + "input_tokens": 47562, + "output_tokens": 61449, + "total_tokens": 109011, + "reasoning_tokens": 58831 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 2227.84, + "avg_input_tokens": 28375.2, + "avg_output_tokens": 30959.2, + "avg_total_tokens": 59334.5, + "avg_turn_count": 7.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 183.54871487617493, - "agent_execution_time": 183.54871487617493, - "token_usage": { - "input_tokens": 77149, - "output_tokens": 3075, - "total_tokens": 80224, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 183.55, - "avg_input_tokens": 77149.0, - "avg_output_tokens": 3075.0, - "avg_total_tokens": 80224.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 779.1507678031921, + "agent_execution_time": 779.1507678031921, + "token_usage": { + "input_tokens": 1813602, + "output_tokens": 25881, + "total_tokens": 1839483, + "reasoning_tokens": 0 + }, + "turn_count": 59 + }, + { + "run": "run-2", + "success": false, + "execution_time": 220.82219648361206, + "agent_execution_time": 220.82219648361206, + "token_usage": { + "input_tokens": 33861, + "output_tokens": 2069, + "total_tokens": 35930, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 35.50087571144104, + "agent_execution_time": 35.50087571144104, + "token_usage": { + "input_tokens": 27596, + "output_tokens": 1922, + "total_tokens": 29518, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 34.65121340751648, + "agent_execution_time": 34.65121340751648, + "token_usage": { + "input_tokens": 27477, + "output_tokens": 1877, + "total_tokens": 29354, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 267.53, + "avg_input_tokens": 475634.0, + "avg_output_tokens": 7937.2, + "avg_total_tokens": 483571.2, + "avg_turn_count": 20.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 263.3165783882141, + "agent_execution_time": 263.3165783882141, + "token_usage": { + "input_tokens": 1184163, + "output_tokens": 11529, + "total_tokens": 1195692, + "reasoning_tokens": 0 + }, + "turn_count": 92 + }, + { + "run": "run-2", + "success": false, + "execution_time": 113.75364208221436, + "agent_execution_time": 113.75364208221436, + "token_usage": { + "input_tokens": 394780, + "output_tokens": 6695, + "total_tokens": 401475, + "reasoning_tokens": 0 + }, + "turn_count": 50 + }, + { + "run": "run-3", + "success": false, + "execution_time": 263.3165783882141, + "agent_execution_time": 263.3165783882141, + "token_usage": { + "input_tokens": 1184163, + "output_tokens": 11529, + "total_tokens": 1195692, + "reasoning_tokens": 0 + }, + "turn_count": 92 + }, + { + "run": "run-4", + "success": false, + "execution_time": 113.75364208221436, + "agent_execution_time": 113.75364208221436, + "token_usage": { + "input_tokens": 394780, + "output_tokens": 6695, + "total_tokens": 401475, + "reasoning_tokens": 0 + }, + "turn_count": 50 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 188.54, + "avg_input_tokens": 789471.5, + "avg_output_tokens": 9112.0, + "avg_total_tokens": 798583.5, + "avg_turn_count": 71.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 55.424028158187866, + "agent_execution_time": 55.424028158187866, + "token_usage": { + "input_tokens": 19435, + "output_tokens": 671, + "total_tokens": 20106, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 54.62150979042053, + "agent_execution_time": 54.62150979042053, + "token_usage": { + "input_tokens": 19412, + "output_tokens": 660, + "total_tokens": 20072, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 150.56407022476196, + "agent_execution_time": 150.56407022476196, + "token_usage": { + "input_tokens": 43327, + "output_tokens": 3035, + "total_tokens": 46362, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 62.24797868728638, + "agent_execution_time": 62.24797868728638, + "token_usage": { + "input_tokens": 23752, + "output_tokens": 1211, + "total_tokens": 24963, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 80.71, + "avg_input_tokens": 26481.5, + "avg_output_tokens": 1394.2, + "avg_total_tokens": 27875.8, + "avg_turn_count": 7.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 100.13506531715393, + "agent_execution_time": 100.13506531715393, + "token_usage": { + "input_tokens": 31606, + "output_tokens": 9275, + "total_tokens": 40881, + "reasoning_tokens": 8647 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 87.65612268447876, + "agent_execution_time": 87.65612268447876, + "token_usage": { + "input_tokens": 28641, + "output_tokens": 13312, + "total_tokens": 41953, + "reasoning_tokens": 12417 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 78.489187002182, + "agent_execution_time": 78.489187002182, + "token_usage": { + "input_tokens": 17154, + "output_tokens": 7367, + "total_tokens": 24521, + "reasoning_tokens": 7131 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 115.47236323356628, + "agent_execution_time": 115.47236323356628, + "token_usage": { + "input_tokens": 50185, + "output_tokens": 9795, + "total_tokens": 59980, + "reasoning_tokens": 8541 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 95.44, + "avg_input_tokens": 31896.5, + "avg_output_tokens": 9937.2, + "avg_total_tokens": 41833.8, + "avg_turn_count": 7.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/file_context__pattern_matching.json b/task_results/file_context__pattern_matching.json index 30dd9061..8ab2468e 100644 --- a/task_results/file_context__pattern_matching.json +++ b/task_results/file_context__pattern_matching.json @@ -2,6 +2,73 @@ "task": "file_context__pattern_matching", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 105.45835661888123, + "agent_execution_time": 105.45835661888123, + "token_usage": { + "input_tokens": 26734, + "output_tokens": 1853, + "total_tokens": 28587, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 1728.1030645370483, + "agent_execution_time": 1728.1030645370483, + "token_usage": { + "input_tokens": 31823, + "output_tokens": 31964, + "total_tokens": 63787, + "reasoning_tokens": 30866 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 883.2345929145813, + "agent_execution_time": 883.2345929145813, + "token_usage": { + "input_tokens": 24442, + "output_tokens": 23258, + "total_tokens": 47700, + "reasoning_tokens": 22192 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": true, + "execution_time": 3167.353559732437, + "agent_execution_time": 3167.353559732437, + "token_usage": { + "input_tokens": 69485, + "output_tokens": 91934, + "total_tokens": 161419, + "reasoning_tokens": 90362 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 1471.04, + "avg_input_tokens": 38121.0, + "avg_output_tokens": 37252.2, + "avg_total_tokens": 75373.2, + "avg_turn_count": 7.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 130.5777690410614, - "agent_execution_time": 130.5777690410614, - "token_usage": { - "input_tokens": 27455, - "output_tokens": 2421, - "total_tokens": 29876, - "reasoning_tokens": null - }, - "turn_count": 7 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 130.58, - "avg_input_tokens": 27455.0, - "avg_output_tokens": 2421.0, - "avg_total_tokens": 29876.0, - "avg_turn_count": 7.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 58.629934549331665, + "agent_execution_time": 58.629934549331665, + "token_usage": { + "input_tokens": 36762, + "output_tokens": 2859, + "total_tokens": 39621, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 48.12376308441162, + "agent_execution_time": 48.12376308441162, + "token_usage": { + "input_tokens": 30432, + "output_tokens": 2844, + "total_tokens": 33276, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 40.02613186836243, + "agent_execution_time": 40.02613186836243, + "token_usage": { + "input_tokens": 29981, + "output_tokens": 2330, + "total_tokens": 32311, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 40.59089994430542, + "agent_execution_time": 40.59089994430542, + "token_usage": { + "input_tokens": 29908, + "output_tokens": 2282, + "total_tokens": 32190, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 46.84, + "avg_input_tokens": 31770.8, + "avg_output_tokens": 2578.8, + "avg_total_tokens": 34349.5, + "avg_turn_count": 7.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 108.85073900222778, + "agent_execution_time": 108.85073900222778, + "token_usage": { + "input_tokens": 184509, + "output_tokens": 3979, + "total_tokens": 188488, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-2", + "success": false, + "execution_time": 34.216456174850464, + "agent_execution_time": 34.216456174850464, + "token_usage": { + "input_tokens": 41998, + "output_tokens": 3474, + "total_tokens": 45472, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 108.85073900222778, + "agent_execution_time": 108.85073900222778, + "token_usage": { + "input_tokens": 184509, + "output_tokens": 3979, + "total_tokens": 188488, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": false, + "execution_time": 34.216456174850464, + "agent_execution_time": 34.216456174850464, + "token_usage": { + "input_tokens": 41998, + "output_tokens": 3474, + "total_tokens": 45472, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 71.53, + "avg_input_tokens": 113253.5, + "avg_output_tokens": 3726.5, + "avg_total_tokens": 116980.0, + "avg_turn_count": 19.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 119.15594458580017, + "agent_execution_time": 119.15594458580017, + "token_usage": { + "input_tokens": 38214, + "output_tokens": 1976, + "total_tokens": 40190, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": true, + "execution_time": 144.82859301567078, + "agent_execution_time": 144.82859301567078, + "token_usage": { + "input_tokens": 64496, + "output_tokens": 2165, + "total_tokens": 66661, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 70.72907853126526, + "agent_execution_time": 70.72907853126526, + "token_usage": { + "input_tokens": 24657, + "output_tokens": 967, + "total_tokens": 25624, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 66.93589568138123, + "agent_execution_time": 66.93589568138123, + "token_usage": { + "input_tokens": 35601, + "output_tokens": 1196, + "total_tokens": 36797, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 100.41, + "avg_input_tokens": 40742.0, + "avg_output_tokens": 1576.0, + "avg_total_tokens": 42318.0, + "avg_turn_count": 9.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 82.91513752937317, + "agent_execution_time": 82.91513752937317, + "token_usage": { + "input_tokens": 29557, + "output_tokens": 7313, + "total_tokens": 36870, + "reasoning_tokens": 6449 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 58.620314598083496, + "agent_execution_time": 58.620314598083496, + "token_usage": { + "input_tokens": 30052, + "output_tokens": 7533, + "total_tokens": 37585, + "reasoning_tokens": 6669 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 73.68318223953247, + "agent_execution_time": 73.68318223953247, + "token_usage": { + "input_tokens": 30385, + "output_tokens": 8142, + "total_tokens": 38527, + "reasoning_tokens": 7278 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 63.82978868484497, + "agent_execution_time": 63.82978868484497, + "token_usage": { + "input_tokens": 30232, + "output_tokens": 7012, + "total_tokens": 37244, + "reasoning_tokens": 6148 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 69.76, + "avg_input_tokens": 30056.5, + "avg_output_tokens": 7500.0, + "avg_total_tokens": 37556.5, + "avg_turn_count": 7.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/file_context__uppercase.json b/task_results/file_context__uppercase.json index 13f0fa0d..3c1a70d7 100644 --- a/task_results/file_context__uppercase.json +++ b/task_results/file_context__uppercase.json @@ -2,6 +2,73 @@ "task": "file_context__uppercase", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 147.84447693824768, + "agent_execution_time": 147.84447693824768, + "token_usage": { + "input_tokens": 80409, + "output_tokens": 1927, + "total_tokens": 82336, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 699.920937538147, + "agent_execution_time": 699.920937538147, + "token_usage": { + "input_tokens": 28712, + "output_tokens": 19449, + "total_tokens": 48161, + "reasoning_tokens": 17523 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 980.858642578125, + "agent_execution_time": 980.858642578125, + "token_usage": { + "input_tokens": 28715, + "output_tokens": 25206, + "total_tokens": 53921, + "reasoning_tokens": 23197 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": true, + "execution_time": 617.2144229412079, + "agent_execution_time": 617.2144229412079, + "token_usage": { + "input_tokens": 17251, + "output_tokens": 17797, + "total_tokens": 35048, + "reasoning_tokens": 15882 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 611.46, + "avg_input_tokens": 38771.8, + "avg_output_tokens": 16094.8, + "avg_total_tokens": 54866.5, + "avg_turn_count": 9.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 195.87972903251648, - "agent_execution_time": 195.87972903251648, - "token_usage": { - "input_tokens": 85121, - "output_tokens": 3663, - "total_tokens": 88784, - "reasoning_tokens": null - }, - "turn_count": 17 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 195.88, - "avg_input_tokens": 85121.0, - "avg_output_tokens": 3663.0, - "avg_total_tokens": 88784.0, - "avg_turn_count": 17.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 64.87658524513245, + "agent_execution_time": 64.87658524513245, + "token_usage": { + "input_tokens": 47873, + "output_tokens": 3281, + "total_tokens": 51154, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 53.93935751914978, + "agent_execution_time": 53.93935751914978, + "token_usage": { + "input_tokens": 48465, + "output_tokens": 3199, + "total_tokens": 51664, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 55.271063566207886, + "agent_execution_time": 55.271063566207886, + "token_usage": { + "input_tokens": 57162, + "output_tokens": 3278, + "total_tokens": 60440, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 52.23234415054321, + "agent_execution_time": 52.23234415054321, + "token_usage": { + "input_tokens": 48598, + "output_tokens": 3276, + "total_tokens": 51874, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 56.58, + "avg_input_tokens": 50524.5, + "avg_output_tokens": 3258.5, + "avg_total_tokens": 53783.0, + "avg_turn_count": 9.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 42.0013427734375, + "agent_execution_time": 42.0013427734375, + "token_usage": { + "input_tokens": 75119, + "output_tokens": 2687, + "total_tokens": 77806, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 38.24220895767212, + "agent_execution_time": 38.24220895767212, + "token_usage": { + "input_tokens": 80457, + "output_tokens": 2638, + "total_tokens": 83095, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 42.0013427734375, + "agent_execution_time": 42.0013427734375, + "token_usage": { + "input_tokens": 75119, + "output_tokens": 2687, + "total_tokens": 77806, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 38.24220895767212, + "agent_execution_time": 38.24220895767212, + "token_usage": { + "input_tokens": 80457, + "output_tokens": 2638, + "total_tokens": 83095, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 40.12, + "avg_input_tokens": 77788.0, + "avg_output_tokens": 2662.5, + "avg_total_tokens": 80450.5, + "avg_turn_count": 17.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 161.1598241329193, + "agent_execution_time": 161.1598241329193, + "token_usage": { + "input_tokens": 81410, + "output_tokens": 2271, + "total_tokens": 83681, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 161.52420949935913, + "agent_execution_time": 161.52420949935913, + "token_usage": { + "input_tokens": 80667, + "output_tokens": 2019, + "total_tokens": 82686, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 102.55107808113098, + "agent_execution_time": 102.55107808113098, + "token_usage": { + "input_tokens": 38517, + "output_tokens": 1981, + "total_tokens": 40498, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 100.03137135505676, + "agent_execution_time": 100.03137135505676, + "token_usage": { + "input_tokens": 34960, + "output_tokens": 2057, + "total_tokens": 37017, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 131.32, + "avg_input_tokens": 58888.5, + "avg_output_tokens": 2082.0, + "avg_total_tokens": 60970.5, + "avg_turn_count": 12.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 32.156017541885376, + "agent_execution_time": 32.156017541885376, + "token_usage": { + "input_tokens": 30410, + "output_tokens": 4149, + "total_tokens": 34559, + "reasoning_tokens": 2543 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 21.8786678314209, + "agent_execution_time": 21.8786678314209, + "token_usage": { + "input_tokens": 32069, + "output_tokens": 3201, + "total_tokens": 35270, + "reasoning_tokens": 1602 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 25.078603506088257, + "agent_execution_time": 25.078603506088257, + "token_usage": { + "input_tokens": 21175, + "output_tokens": 3077, + "total_tokens": 24252, + "reasoning_tokens": 1463 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 32.34539723396301, + "agent_execution_time": 32.34539723396301, + "token_usage": { + "input_tokens": 31591, + "output_tokens": 4366, + "total_tokens": 35957, + "reasoning_tokens": 2743 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 27.86, + "avg_input_tokens": 28811.2, + "avg_output_tokens": 3698.2, + "avg_total_tokens": 32509.5, + "avg_turn_count": 7.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/file_property__size_classification.json b/task_results/file_property__size_classification.json index 232c9ec0..aee72b4d 100644 --- a/task_results/file_property__size_classification.json +++ b/task_results/file_property__size_classification.json @@ -2,6 +2,73 @@ "task": "file_property__size_classification", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 118.75910472869873, + "agent_execution_time": 118.75910472869873, + "token_usage": { + "input_tokens": 35182, + "output_tokens": 2097, + "total_tokens": 37279, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": true, + "execution_time": 346.4201979637146, + "agent_execution_time": 346.4201979637146, + "token_usage": { + "input_tokens": 22003, + "output_tokens": 8690, + "total_tokens": 30693, + "reasoning_tokens": 7161 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": true, + "execution_time": 483.41164898872375, + "agent_execution_time": 483.41164898872375, + "token_usage": { + "input_tokens": 27939, + "output_tokens": 11448, + "total_tokens": 39387, + "reasoning_tokens": 9950 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": true, + "execution_time": 351.00888657569885, + "agent_execution_time": 351.00888657569885, + "token_usage": { + "input_tokens": 33871, + "output_tokens": 8957, + "total_tokens": 42828, + "reasoning_tokens": 7094 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 324.9, + "avg_input_tokens": 29748.8, + "avg_output_tokens": 7798.0, + "avg_total_tokens": 37546.8, + "avg_turn_count": 6.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 158.52210807800293, - "agent_execution_time": 158.52210807800293, - "token_usage": { - "input_tokens": 66506, - "output_tokens": 2839, - "total_tokens": 69345, - "reasoning_tokens": null - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 158.52, - "avg_input_tokens": 66506.0, - "avg_output_tokens": 2839.0, - "avg_total_tokens": 69345.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 104.95841813087463, + "agent_execution_time": 104.95841813087463, + "token_usage": { + "input_tokens": 50594, + "output_tokens": 2909, + "total_tokens": 53503, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": true, + "execution_time": 42.95197820663452, + "agent_execution_time": 42.95197820663452, + "token_usage": { + "input_tokens": 37903, + "output_tokens": 2914, + "total_tokens": 40817, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": true, + "execution_time": 48.14620399475098, + "agent_execution_time": 48.14620399475098, + "token_usage": { + "input_tokens": 48099, + "output_tokens": 3176, + "total_tokens": 51275, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": true, + "execution_time": 41.206560373306274, + "agent_execution_time": 41.206560373306274, + "token_usage": { + "input_tokens": 38015, + "output_tokens": 2944, + "total_tokens": 40959, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 59.32, + "avg_input_tokens": 43652.8, + "avg_output_tokens": 2985.8, + "avg_total_tokens": 46638.5, + "avg_turn_count": 7.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 67.11552572250366, + "agent_execution_time": 67.11552572250366, + "token_usage": { + "input_tokens": 136581, + "output_tokens": 3459, + "total_tokens": 140040, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-2", + "success": true, + "execution_time": 29.143349409103394, + "agent_execution_time": 29.143349409103394, + "token_usage": { + "input_tokens": 66522, + "output_tokens": 2046, + "total_tokens": 68568, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 67.11552572250366, + "agent_execution_time": 67.11552572250366, + "token_usage": { + "input_tokens": 136581, + "output_tokens": 3459, + "total_tokens": 140040, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": true, + "execution_time": 29.143349409103394, + "agent_execution_time": 29.143349409103394, + "token_usage": { + "input_tokens": 66522, + "output_tokens": 2046, + "total_tokens": 68568, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 48.13, + "avg_input_tokens": 101551.5, + "avg_output_tokens": 2752.5, + "avg_total_tokens": 104304.0, + "avg_turn_count": 23.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 120.33503413200378, + "agent_execution_time": 120.33503413200378, + "token_usage": { + "input_tokens": 35190, + "output_tokens": 2085, + "total_tokens": 37275, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": true, + "execution_time": 122.93121337890625, + "agent_execution_time": 122.93121337890625, + "token_usage": { + "input_tokens": 35115, + "output_tokens": 2033, + "total_tokens": 37148, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": true, + "execution_time": 133.4520697593689, + "agent_execution_time": 133.4520697593689, + "token_usage": { + "input_tokens": 67018, + "output_tokens": 2212, + "total_tokens": 69230, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": false, + "execution_time": 5.349783182144165, + "agent_execution_time": 5.349783182144165, + "token_usage": { + "input_tokens": 3026, + "output_tokens": 57, + "total_tokens": 3083, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 95.52, + "avg_input_tokens": 35087.2, + "avg_output_tokens": 1596.8, + "avg_total_tokens": 36684.0, + "avg_turn_count": 7.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 25.813899993896484, + "agent_execution_time": 25.813899993896484, + "token_usage": { + "input_tokens": 26934, + "output_tokens": 2779, + "total_tokens": 29713, + "reasoning_tokens": 1161 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 18.65298867225647, + "agent_execution_time": 18.65298867225647, + "token_usage": { + "input_tokens": 19742, + "output_tokens": 2351, + "total_tokens": 22093, + "reasoning_tokens": 1235 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": true, + "execution_time": 16.838480234146118, + "agent_execution_time": 16.838480234146118, + "token_usage": { + "input_tokens": 19408, + "output_tokens": 2060, + "total_tokens": 21468, + "reasoning_tokens": 921 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": true, + "execution_time": 24.74107313156128, + "agent_execution_time": 24.74107313156128, + "token_usage": { + "input_tokens": 31108, + "output_tokens": 2732, + "total_tokens": 33840, + "reasoning_tokens": 1064 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 21.51, + "avg_input_tokens": 24298.0, + "avg_output_tokens": 2480.5, + "avg_total_tokens": 26778.5, + "avg_turn_count": 6.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/file_property__time_classification.json b/task_results/file_property__time_classification.json index 0773f252..0511e77f 100644 --- a/task_results/file_property__time_classification.json +++ b/task_results/file_property__time_classification.json @@ -2,6 +2,73 @@ "task": "file_property__time_classification", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 177.63192105293274, + "agent_execution_time": 177.63192105293274, + "token_usage": { + "input_tokens": 107155, + "output_tokens": 2069, + "total_tokens": 109224, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 800.0584149360657, + "agent_execution_time": 800.0584149360657, + "token_usage": { + "input_tokens": 31385, + "output_tokens": 20574, + "total_tokens": 51959, + "reasoning_tokens": 17946 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 743.9570920467377, + "agent_execution_time": 743.9570920467377, + "token_usage": { + "input_tokens": 30651, + "output_tokens": 18341, + "total_tokens": 48992, + "reasoning_tokens": 15712 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 627.0146262645721, + "agent_execution_time": 627.0146262645721, + "token_usage": { + "input_tokens": 41123, + "output_tokens": 17698, + "total_tokens": 58821, + "reasoning_tokens": 14092 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 587.17, + "avg_input_tokens": 52578.5, + "avg_output_tokens": 14670.5, + "avg_total_tokens": 67249.0, + "avg_turn_count": 10.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 301.9879250526428, - "agent_execution_time": 301.9879250526428, - "token_usage": { - "input_tokens": 187540, - "output_tokens": 5139, - "total_tokens": 192679, - "reasoning_tokens": null - }, - "turn_count": 32 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 301.99, - "avg_input_tokens": 187540.0, - "avg_output_tokens": 5139.0, - "avg_total_tokens": 192679.0, - "avg_turn_count": 32.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 74.05369782447815, + "agent_execution_time": 74.05369782447815, + "token_usage": { + "input_tokens": 80923, + "output_tokens": 3915, + "total_tokens": 84838, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 70.1641845703125, + "agent_execution_time": 70.1641845703125, + "token_usage": { + "input_tokens": 68059, + "output_tokens": 3158, + "total_tokens": 71217, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 52.32022547721863, + "agent_execution_time": 52.32022547721863, + "token_usage": { + "input_tokens": 59222, + "output_tokens": 3045, + "total_tokens": 62267, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 63.27241516113281, + "agent_execution_time": 63.27241516113281, + "token_usage": { + "input_tokens": 75572, + "output_tokens": 3790, + "total_tokens": 79362, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 64.95, + "avg_input_tokens": 70944.0, + "avg_output_tokens": 3477.0, + "avg_total_tokens": 74421.0, + "avg_turn_count": 11.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 49.26250123977661, + "agent_execution_time": 49.26250123977661, + "token_usage": { + "input_tokens": 111501, + "output_tokens": 2696, + "total_tokens": 114197, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 50.258808851242065, + "agent_execution_time": 50.258808851242065, + "token_usage": { + "input_tokens": 119795, + "output_tokens": 2766, + "total_tokens": 122561, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": false, + "execution_time": 49.26250123977661, + "agent_execution_time": 49.26250123977661, + "token_usage": { + "input_tokens": 111501, + "output_tokens": 2696, + "total_tokens": 114197, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 50.258808851242065, + "agent_execution_time": 50.258808851242065, + "token_usage": { + "input_tokens": 119795, + "output_tokens": 2766, + "total_tokens": 122561, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 49.76, + "avg_input_tokens": 115648.0, + "avg_output_tokens": 2731.0, + "avg_total_tokens": 118379.0, + "avg_turn_count": 24.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 184.8964228630066, + "agent_execution_time": 184.8964228630066, + "token_usage": { + "input_tokens": 87055, + "output_tokens": 3014, + "total_tokens": 90069, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 135.25379538536072, + "agent_execution_time": 135.25379538536072, + "token_usage": { + "input_tokens": 58749, + "output_tokens": 2003, + "total_tokens": 60752, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 159.91613221168518, + "agent_execution_time": 159.91613221168518, + "token_usage": { + "input_tokens": 46993, + "output_tokens": 3427, + "total_tokens": 50420, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 151.25762367248535, + "agent_execution_time": 151.25762367248535, + "token_usage": { + "input_tokens": 77032, + "output_tokens": 2336, + "total_tokens": 79368, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 157.83, + "avg_input_tokens": 67457.2, + "avg_output_tokens": 2695.0, + "avg_total_tokens": 70152.2, + "avg_turn_count": 12.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 38.066346883773804, + "agent_execution_time": 38.066346883773804, + "token_usage": { + "input_tokens": 58567, + "output_tokens": 6044, + "total_tokens": 64611, + "reasoning_tokens": 3672 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 58.31969094276428, + "agent_execution_time": 58.31969094276428, + "token_usage": { + "input_tokens": 45709, + "output_tokens": 3605, + "total_tokens": 49314, + "reasoning_tokens": 2040 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 29.37819480895996, + "agent_execution_time": 29.37819480895996, + "token_usage": { + "input_tokens": 46212, + "output_tokens": 3699, + "total_tokens": 49911, + "reasoning_tokens": 2134 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 29.231498956680298, + "agent_execution_time": 29.231498956680298, + "token_usage": { + "input_tokens": 60163, + "output_tokens": 3964, + "total_tokens": 64127, + "reasoning_tokens": 2140 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 38.75, + "avg_input_tokens": 52662.8, + "avg_output_tokens": 4328.0, + "avg_total_tokens": 56990.8, + "avg_turn_count": 9.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/folder_structure__structure_analysis.json b/task_results/folder_structure__structure_analysis.json index 06360a73..23e349ea 100644 --- a/task_results/folder_structure__structure_analysis.json +++ b/task_results/folder_structure__structure_analysis.json @@ -2,6 +2,73 @@ "task": "folder_structure__structure_analysis", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 206.6289942264557, + "agent_execution_time": 206.6289942264557, + "token_usage": { + "input_tokens": 400006, + "output_tokens": 2645, + "total_tokens": 402651, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 3339.836974143982, + "agent_execution_time": 3339.836974143982, + "token_usage": { + "input_tokens": 14293, + "output_tokens": 24081, + "total_tokens": 38374, + "reasoning_tokens": 23906 + }, + "turn_count": 3 + }, + { + "run": "run-3", + "success": false, + "execution_time": 3600.0835769176483, + "agent_execution_time": 3600.0835769176483, + "token_usage": { + "input_tokens": 39652, + "output_tokens": 44229, + "total_tokens": 83881, + "reasoning_tokens": 43921 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 2901.658514738083, + "agent_execution_time": 2901.658514738083, + "token_usage": { + "input_tokens": 47788, + "output_tokens": 31581, + "total_tokens": 79369, + "reasoning_tokens": 31052 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 2512.05, + "avg_input_tokens": 125434.8, + "avg_output_tokens": 25634.0, + "avg_total_tokens": 151068.8, + "avg_turn_count": 8.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 146.45751810073853, - "agent_execution_time": 146.45751810073853, - "token_usage": { - "input_tokens": 167036, - "output_tokens": 1881, - "total_tokens": 168917, - "reasoning_tokens": null - }, - "turn_count": 13 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 146.46, - "avg_input_tokens": 167036.0, - "avg_output_tokens": 1881.0, - "avg_total_tokens": 168917.0, - "avg_turn_count": 13.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 221.27812266349792, + "agent_execution_time": 221.27812266349792, + "token_usage": { + "input_tokens": 260402, + "output_tokens": 13428, + "total_tokens": 273830, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 337.2250006198883, + "agent_execution_time": 337.2250006198883, + "token_usage": { + "input_tokens": 564713, + "output_tokens": 21575, + "total_tokens": 586288, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": false, + "execution_time": 280.85084867477417, + "agent_execution_time": 280.85084867477417, + "token_usage": { + "input_tokens": 818881, + "output_tokens": 16340, + "total_tokens": 835221, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": false, + "execution_time": 221.56634426116943, + "agent_execution_time": 221.56634426116943, + "token_usage": { + "input_tokens": 537992, + "output_tokens": 12023, + "total_tokens": 550015, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 265.23, + "avg_input_tokens": 545497.0, + "avg_output_tokens": 15841.5, + "avg_total_tokens": 561338.5, + "avg_turn_count": 23.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 176.78489065170288, + "agent_execution_time": 176.78489065170288, + "token_usage": { + "input_tokens": 731419, + "output_tokens": 7317, + "total_tokens": 738736, + "reasoning_tokens": 0 + }, + "turn_count": 57 + }, + { + "run": "run-2", + "success": false, + "execution_time": 178.56282258033752, + "agent_execution_time": 178.56282258033752, + "token_usage": { + "input_tokens": 682892, + "output_tokens": 7135, + "total_tokens": 690027, + "reasoning_tokens": 0 + }, + "turn_count": 55 + }, + { + "run": "run-3", + "success": false, + "execution_time": 176.78489065170288, + "agent_execution_time": 176.78489065170288, + "token_usage": { + "input_tokens": 731419, + "output_tokens": 7317, + "total_tokens": 738736, + "reasoning_tokens": 0 + }, + "turn_count": 57 + }, + { + "run": "run-4", + "success": false, + "execution_time": 178.56282258033752, + "agent_execution_time": 178.56282258033752, + "token_usage": { + "input_tokens": 682892, + "output_tokens": 7135, + "total_tokens": 690027, + "reasoning_tokens": 0 + }, + "turn_count": 55 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 177.67, + "avg_input_tokens": 707155.5, + "avg_output_tokens": 7226.0, + "avg_total_tokens": 714381.5, + "avg_turn_count": 56.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 146.94360637664795, + "agent_execution_time": 146.94360637664795, + "token_usage": { + "input_tokens": 215097, + "output_tokens": 1574, + "total_tokens": 216671, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 138.36178493499756, + "agent_execution_time": 138.36178493499756, + "token_usage": { + "input_tokens": 307052, + "output_tokens": 1531, + "total_tokens": 308583, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": false, + "execution_time": 53.85977840423584, + "agent_execution_time": 53.85977840423584, + "token_usage": { + "input_tokens": 38737, + "output_tokens": 674, + "total_tokens": 39411, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 73.83413124084473, + "agent_execution_time": 73.83413124084473, + "token_usage": { + "input_tokens": 118718, + "output_tokens": 715, + "total_tokens": 119433, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 103.25, + "avg_input_tokens": 169901.0, + "avg_output_tokens": 1123.5, + "avg_total_tokens": 171024.5, + "avg_turn_count": 10.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 124.99852323532104, + "agent_execution_time": 124.99852323532104, + "token_usage": { + "input_tokens": 542336, + "output_tokens": 18742, + "total_tokens": 561078, + "reasoning_tokens": 17773 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": true, + "execution_time": 216.39523315429688, + "agent_execution_time": 216.39523315429688, + "token_usage": { + "input_tokens": 1586327, + "output_tokens": 24166, + "total_tokens": 1610493, + "reasoning_tokens": 18554 + }, + "turn_count": 67 + }, + { + "run": "run-3", + "success": false, + "execution_time": 178.13117289543152, + "agent_execution_time": 178.13117289543152, + "token_usage": { + "input_tokens": 594382, + "output_tokens": 25546, + "total_tokens": 619928, + "reasoning_tokens": 18822 + }, + "turn_count": 25 + }, + { + "run": "run-4", + "success": false, + "execution_time": 175.06329870224, + "agent_execution_time": 175.06329870224, + "token_usage": { + "input_tokens": 439709, + "output_tokens": 20890, + "total_tokens": 460599, + "reasoning_tokens": 14363 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 173.65, + "avg_input_tokens": 790688.5, + "avg_output_tokens": 22336.0, + "avg_total_tokens": 813024.5, + "avg_turn_count": 32.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/folder_structure__structure_mirror.json b/task_results/folder_structure__structure_mirror.json index efbe9f98..c87d5567 100644 --- a/task_results/folder_structure__structure_mirror.json +++ b/task_results/folder_structure__structure_mirror.json @@ -2,6 +2,73 @@ "task": "folder_structure__structure_mirror", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 603.8157112598419, + "agent_execution_time": 603.8157112598419, + "token_usage": { + "input_tokens": 934563, + "output_tokens": 6244, + "total_tokens": 940807, + "reasoning_tokens": 0 + }, + "turn_count": 75 + }, + { + "run": "run-2", + "success": true, + "execution_time": 1907.7407739162445, + "agent_execution_time": 1907.7407739162445, + "token_usage": { + "input_tokens": 41848, + "output_tokens": 31035, + "total_tokens": 72883, + "reasoning_tokens": 28470 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 2274.7533860206604, + "agent_execution_time": 2274.7533860206604, + "token_usage": { + "input_tokens": 71924, + "output_tokens": 58250, + "total_tokens": 130174, + "reasoning_tokens": 53564 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1513.5469272136688, + "agent_execution_time": 1513.5469272136688, + "token_usage": { + "input_tokens": 54984, + "output_tokens": 43127, + "total_tokens": 98111, + "reasoning_tokens": 40233 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 1574.96, + "avg_input_tokens": 275829.8, + "avg_output_tokens": 34664.0, + "avg_total_tokens": 310493.8, + "avg_turn_count": 24.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 363.83149099349976, - "agent_execution_time": 363.83149099349976, - "token_usage": { - "input_tokens": 354411, - "output_tokens": 5571, - "total_tokens": 359982, - "reasoning_tokens": null - }, - "turn_count": 33 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 363.83, - "avg_input_tokens": 354411.0, - "avg_output_tokens": 5571.0, - "avg_total_tokens": 359982.0, - "avg_turn_count": 33.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 111.14954137802124, + "agent_execution_time": 111.14954137802124, + "token_usage": { + "input_tokens": 231693, + "output_tokens": 5858, + "total_tokens": 237551, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": true, + "execution_time": 230.31520056724548, + "agent_execution_time": 230.31520056724548, + "token_usage": { + "input_tokens": 355034, + "output_tokens": 13517, + "total_tokens": 368551, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": true, + "execution_time": 73.10000491142273, + "agent_execution_time": 73.10000491142273, + "token_usage": { + "input_tokens": 88445, + "output_tokens": 4797, + "total_tokens": 93242, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 162.5775134563446, + "agent_execution_time": 162.5775134563446, + "token_usage": { + "input_tokens": 342432, + "output_tokens": 11913, + "total_tokens": 354345, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 144.29, + "avg_input_tokens": 254401.0, + "avg_output_tokens": 9021.2, + "avg_total_tokens": 263422.2, + "avg_turn_count": 16.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 153.52127957344055, + "agent_execution_time": 153.52127957344055, + "token_usage": { + "input_tokens": 400620, + "output_tokens": 5420, + "total_tokens": 406040, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + { + "run": "run-2", + "success": true, + "execution_time": 109.83824896812439, + "agent_execution_time": 109.83824896812439, + "token_usage": { + "input_tokens": 495318, + "output_tokens": 6048, + "total_tokens": 501366, + "reasoning_tokens": 0 + }, + "turn_count": 47 + }, + { + "run": "run-3", + "success": false, + "execution_time": 153.52127957344055, + "agent_execution_time": 153.52127957344055, + "token_usage": { + "input_tokens": 400620, + "output_tokens": 5420, + "total_tokens": 406040, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + { + "run": "run-4", + "success": true, + "execution_time": 109.83824896812439, + "agent_execution_time": 109.83824896812439, + "token_usage": { + "input_tokens": 495318, + "output_tokens": 6048, + "total_tokens": 501366, + "reasoning_tokens": 0 + }, + "turn_count": 47 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 131.68, + "avg_input_tokens": 447969.0, + "avg_output_tokens": 5734.0, + "avg_total_tokens": 453703.0, + "avg_turn_count": 43.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 369.82984828948975, + "agent_execution_time": 369.82984828948975, + "token_usage": { + "input_tokens": 436585, + "output_tokens": 4163, + "total_tokens": 440748, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + { + "run": "run-2", + "success": false, + "execution_time": 342.6927993297577, + "agent_execution_time": 342.6927993297577, + "token_usage": { + "input_tokens": 371846, + "output_tokens": 4118, + "total_tokens": 375964, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + { + "run": "run-3", + "success": false, + "execution_time": 306.3251585960388, + "agent_execution_time": 306.3251585960388, + "token_usage": { + "input_tokens": 263742, + "output_tokens": 2713, + "total_tokens": 266455, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": false, + "execution_time": 114.73742127418518, + "agent_execution_time": 114.73742127418518, + "token_usage": { + "input_tokens": 84609, + "output_tokens": 2084, + "total_tokens": 86693, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 283.4, + "avg_input_tokens": 289195.5, + "avg_output_tokens": 3269.5, + "avg_total_tokens": 292465.0, + "avg_turn_count": 30.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 58.23767924308777, + "agent_execution_time": 58.23767924308777, + "token_usage": { + "input_tokens": 101245, + "output_tokens": 6458, + "total_tokens": 107703, + "reasoning_tokens": 4059 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 95.15689301490784, + "agent_execution_time": 95.15689301490784, + "token_usage": { + "input_tokens": 483449, + "output_tokens": 6326, + "total_tokens": 489775, + "reasoning_tokens": 2935 + }, + "turn_count": 43 + }, + { + "run": "run-3", + "success": true, + "execution_time": 69.45335984230042, + "agent_execution_time": 69.45335984230042, + "token_usage": { + "input_tokens": 146171, + "output_tokens": 8389, + "total_tokens": 154560, + "reasoning_tokens": 5453 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 68.67535305023193, + "agent_execution_time": 68.67535305023193, + "token_usage": { + "input_tokens": 120824, + "output_tokens": 8657, + "total_tokens": 129481, + "reasoning_tokens": 5892 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 72.88, + "avg_input_tokens": 212922.2, + "avg_output_tokens": 7457.5, + "avg_total_tokens": 220379.8, + "avg_turn_count": 20.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/harmony__fix_conflict.json b/task_results/harmony__fix_conflict.json index b0d5e048..b9cd1e70 100644 --- a/task_results/harmony__fix_conflict.json +++ b/task_results/harmony__fix_conflict.json @@ -2,6 +2,73 @@ "task": "harmony__fix_conflict", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 331.02082896232605, + "agent_execution_time": 331.02082896232605, + "token_usage": { + "input_tokens": 1614626, + "output_tokens": 2797, + "total_tokens": 1617423, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": false, + "execution_time": 641.3220250606537, + "agent_execution_time": 641.3220250606537, + "token_usage": { + "input_tokens": 493963, + "output_tokens": 5939, + "total_tokens": 499902, + "reasoning_tokens": 5520 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 346.60584688186646, + "agent_execution_time": 346.60584688186646, + "token_usage": { + "input_tokens": 257396, + "output_tokens": 5998, + "total_tokens": 263394, + "reasoning_tokens": 5592 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 133.6654496192932, + "agent_execution_time": 133.6654496192932, + "token_usage": { + "input_tokens": 141244, + "output_tokens": 3187, + "total_tokens": 144431, + "reasoning_tokens": 3047 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 363.15, + "avg_input_tokens": 626807.2, + "avg_output_tokens": 4480.2, + "avg_total_tokens": 631287.5, + "avg_turn_count": 9.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 382.32449102401733, - "agent_execution_time": 382.32449102401733, - "token_usage": { - "input_tokens": 869307, - "output_tokens": 4839, - "total_tokens": 874146, - "reasoning_tokens": null - }, - "turn_count": 14 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 382.32, - "avg_input_tokens": 869307.0, - "avg_output_tokens": 4839.0, - "avg_total_tokens": 874146.0, - "avg_turn_count": 14.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 271.94125032424927, + "agent_execution_time": 271.94125032424927, + "token_usage": { + "input_tokens": 3238218, + "output_tokens": 6750, + "total_tokens": 3244968, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-2", + "success": false, + "execution_time": 231.37410593032837, + "agent_execution_time": 231.37410593032837, + "token_usage": { + "input_tokens": 2520756, + "output_tokens": 6402, + "total_tokens": 2527158, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 381.10024642944336, + "agent_execution_time": 381.10024642944336, + "token_usage": { + "input_tokens": 3948832, + "output_tokens": 7663, + "total_tokens": 3956495, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-4", + "success": false, + "execution_time": 233.51602387428284, + "agent_execution_time": 233.51602387428284, + "token_usage": { + "input_tokens": 2641556, + "output_tokens": 5286, + "total_tokens": 2646842, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 279.48, + "avg_input_tokens": 3087340.5, + "avg_output_tokens": 6525.2, + "avg_total_tokens": 3093865.8, + "avg_turn_count": 28.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 196.27353477478027, + "agent_execution_time": 196.27353477478027, + "token_usage": { + "input_tokens": 1336398, + "output_tokens": 3498, + "total_tokens": 1339896, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": false, + "execution_time": 231.27164697647095, + "agent_execution_time": 231.27164697647095, + "token_usage": { + "input_tokens": 1768764, + "output_tokens": 2841, + "total_tokens": 1771605, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-3", + "success": false, + "execution_time": 263.36326789855957, + "agent_execution_time": 263.36326789855957, + "token_usage": { + "input_tokens": 2244879, + "output_tokens": 4397, + "total_tokens": 2249276, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + { + "run": "run-4", + "success": false, + "execution_time": 278.4939820766449, + "agent_execution_time": 278.4939820766449, + "token_usage": { + "input_tokens": 3126223, + "output_tokens": 6434, + "total_tokens": 3132657, + "reasoning_tokens": 0 + }, + "turn_count": 37 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 242.35, + "avg_input_tokens": 2119066.0, + "avg_output_tokens": 4292.5, + "avg_total_tokens": 2123358.5, + "avg_turn_count": 32.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 293.03666639328003, + "agent_execution_time": 293.03666639328003, + "token_usage": { + "input_tokens": 1406576, + "output_tokens": 2583, + "total_tokens": 1409159, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.470372438430786, + "agent_execution_time": 10.470372438430786, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 768.0327546596527, + "agent_execution_time": 768.0327546596527, + "token_usage": { + "input_tokens": 1302560, + "output_tokens": 2559, + "total_tokens": 1305119, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 245.54946327209473, + "agent_execution_time": 245.54946327209473, + "token_usage": { + "input_tokens": 573912, + "output_tokens": 979, + "total_tokens": 574891, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 329.27, + "avg_input_tokens": 820762.0, + "avg_output_tokens": 1530.2, + "avg_total_tokens": 822292.2, + "avg_turn_count": 12.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 158.0662121772766, + "agent_execution_time": 158.0662121772766, + "token_usage": { + "input_tokens": 1278048, + "output_tokens": 9931, + "total_tokens": 1287979, + "reasoning_tokens": 3607 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 132.37931084632874, + "agent_execution_time": 132.37931084632874, + "token_usage": { + "input_tokens": 956874, + "output_tokens": 8651, + "total_tokens": 965525, + "reasoning_tokens": 2948 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 119.94980001449585, + "agent_execution_time": 119.94980001449585, + "token_usage": { + "input_tokens": 1865531, + "output_tokens": 7148, + "total_tokens": 1872679, + "reasoning_tokens": 5271 + }, + "turn_count": 25 + }, + { + "run": "run-4", + "success": false, + "execution_time": 121.68680691719055, + "agent_execution_time": 121.68680691719055, + "token_usage": { + "input_tokens": 1233935, + "output_tokens": 7964, + "total_tokens": 1241899, + "reasoning_tokens": 6046 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 133.02, + "avg_input_tokens": 1333597.0, + "avg_output_tokens": 8423.5, + "avg_total_tokens": 1342020.5, + "avg_turn_count": 18.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/harmony__issue_pr_commit_workflow.json b/task_results/harmony__issue_pr_commit_workflow.json index c033e5f9..2d710406 100644 --- a/task_results/harmony__issue_pr_commit_workflow.json +++ b/task_results/harmony__issue_pr_commit_workflow.json @@ -2,6 +2,73 @@ "task": "harmony__issue_pr_commit_workflow", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 157.60750436782837, + "agent_execution_time": 157.60750436782837, + "token_usage": { + "input_tokens": 191573, + "output_tokens": 2282, + "total_tokens": 193855, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 185.31686425209045, + "agent_execution_time": 185.31686425209045, + "token_usage": { + "input_tokens": 24220, + "output_tokens": 4053, + "total_tokens": 28273, + "reasoning_tokens": 3996 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": false, + "execution_time": 546.4371287822723, + "agent_execution_time": 546.4371287822723, + "token_usage": { + "input_tokens": 109899, + "output_tokens": 11975, + "total_tokens": 121874, + "reasoning_tokens": 9271 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 587.9246520996094, + "agent_execution_time": 587.9246520996094, + "token_usage": { + "input_tokens": 112335, + "output_tokens": 16781, + "total_tokens": 129116, + "reasoning_tokens": 13917 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 369.32, + "avg_input_tokens": 109506.8, + "avg_output_tokens": 8772.8, + "avg_total_tokens": 118279.5, + "avg_turn_count": 4.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 267.2391390800476, - "agent_execution_time": 267.2391390800476, - "token_usage": { - "input_tokens": 274885, - "output_tokens": 3532, - "total_tokens": 278417, - "reasoning_tokens": null - }, - "turn_count": 11 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 267.24, - "avg_input_tokens": 274885.0, - "avg_output_tokens": 3532.0, - "avg_total_tokens": 278417.0, - "avg_turn_count": 11.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 95.93805837631226, + "agent_execution_time": 95.93805837631226, + "token_usage": { + "input_tokens": 219281, + "output_tokens": 3445, + "total_tokens": 222726, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 93.80944776535034, + "agent_execution_time": 93.80944776535034, + "token_usage": { + "input_tokens": 218845, + "output_tokens": 3301, + "total_tokens": 222146, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 98.31169533729553, + "agent_execution_time": 98.31169533729553, + "token_usage": { + "input_tokens": 209796, + "output_tokens": 3487, + "total_tokens": 213283, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 119.56449246406555, + "agent_execution_time": 119.56449246406555, + "token_usage": { + "input_tokens": 306150, + "output_tokens": 3870, + "total_tokens": 310020, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 101.91, + "avg_input_tokens": 238518.0, + "avg_output_tokens": 3525.8, + "avg_total_tokens": 242043.8, + "avg_turn_count": 10.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 110.16259336471558, + "agent_execution_time": 110.16259336471558, + "token_usage": { + "input_tokens": 367574, + "output_tokens": 2748, + "total_tokens": 370322, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 502.2443449497223, + "agent_execution_time": 502.2443449497223, + "token_usage": { + "input_tokens": 1370757, + "output_tokens": 8712, + "total_tokens": 1379469, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 142.06048369407654, + "agent_execution_time": 142.06048369407654, + "token_usage": { + "input_tokens": 523834, + "output_tokens": 2476, + "total_tokens": 526310, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": true, + "execution_time": 118.41282606124878, + "agent_execution_time": 118.41282606124878, + "token_usage": { + "input_tokens": 486304, + "output_tokens": 3438, + "total_tokens": 489742, + "reasoning_tokens": 0 + }, + "turn_count": 16 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 218.22, + "avg_input_tokens": 687117.2, + "avg_output_tokens": 4343.5, + "avg_total_tokens": 691460.8, + "avg_turn_count": 16.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 163.6453673839569, + "agent_execution_time": 163.6453673839569, + "token_usage": { + "input_tokens": 191791, + "output_tokens": 2299, + "total_tokens": 194090, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 9.889934539794922, + "agent_execution_time": 9.889934539794922, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 205.85223817825317, + "agent_execution_time": 205.85223817825317, + "token_usage": { + "input_tokens": 263992, + "output_tokens": 2187, + "total_tokens": 266179, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 209.96171307563782, + "agent_execution_time": 209.96171307563782, + "token_usage": { + "input_tokens": 316421, + "output_tokens": 2607, + "total_tokens": 319028, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 147.34, + "avg_input_tokens": 193051.0, + "avg_output_tokens": 1773.2, + "avg_total_tokens": 194824.2, + "avg_turn_count": 8.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 57.13207173347473, + "agent_execution_time": 57.13207173347473, + "token_usage": { + "input_tokens": 206076, + "output_tokens": 2795, + "total_tokens": 208871, + "reasoning_tokens": 1547 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 51.34369421005249, + "agent_execution_time": 51.34369421005249, + "token_usage": { + "input_tokens": 186147, + "output_tokens": 3588, + "total_tokens": 189735, + "reasoning_tokens": 2096 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 50.264371395111084, + "agent_execution_time": 50.264371395111084, + "token_usage": { + "input_tokens": 167917, + "output_tokens": 3125, + "total_tokens": 171042, + "reasoning_tokens": 1896 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 50.642306327819824, + "agent_execution_time": 50.642306327819824, + "token_usage": { + "input_tokens": 229472, + "output_tokens": 3259, + "total_tokens": 232731, + "reasoning_tokens": 1684 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 52.35, + "avg_input_tokens": 197403.0, + "avg_output_tokens": 3191.8, + "avg_total_tokens": 200594.8, + "avg_turn_count": 11.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/harmony__issue_tagging_pr_closure.json b/task_results/harmony__issue_tagging_pr_closure.json index 8248e936..2a41db76 100644 --- a/task_results/harmony__issue_tagging_pr_closure.json +++ b/task_results/harmony__issue_tagging_pr_closure.json @@ -2,6 +2,73 @@ "task": "harmony__issue_tagging_pr_closure", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 196.77281999588013, + "agent_execution_time": 196.77281999588013, + "token_usage": { + "input_tokens": 250367, + "output_tokens": 2665, + "total_tokens": 253032, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 883.2635576725006, + "agent_execution_time": 883.2635576725006, + "token_usage": { + "input_tokens": 236819, + "output_tokens": 17714, + "total_tokens": 254533, + "reasoning_tokens": 15680 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 69.26510071754456, + "agent_execution_time": 69.26510071754456, + "token_usage": { + "input_tokens": 24341, + "output_tokens": 1566, + "total_tokens": 25907, + "reasoning_tokens": 1239 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 303.78307580947876, + "agent_execution_time": 303.78307580947876, + "token_usage": { + "input_tokens": 91207, + "output_tokens": 8048, + "total_tokens": 99255, + "reasoning_tokens": 5073 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 363.27, + "avg_input_tokens": 150683.5, + "avg_output_tokens": 7498.2, + "avg_total_tokens": 158181.8, + "avg_turn_count": 6.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 395.47656083106995, - "agent_execution_time": 395.47656083106995, - "token_usage": { - "input_tokens": 453282, - "output_tokens": 4998, - "total_tokens": 458280, - "reasoning_tokens": null - }, - "turn_count": 15 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 395.48, - "avg_input_tokens": 453282.0, - "avg_output_tokens": 4998.0, - "avg_total_tokens": 458280.0, - "avg_turn_count": 15.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 180.46537899971008, + "agent_execution_time": 180.46537899971008, + "token_usage": { + "input_tokens": 367451, + "output_tokens": 6002, + "total_tokens": 373453, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 175.22188591957092, + "agent_execution_time": 175.22188591957092, + "token_usage": { + "input_tokens": 338439, + "output_tokens": 6445, + "total_tokens": 344884, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": true, + "execution_time": 151.25655937194824, + "agent_execution_time": 151.25655937194824, + "token_usage": { + "input_tokens": 308568, + "output_tokens": 4979, + "total_tokens": 313547, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": true, + "execution_time": 155.67875576019287, + "agent_execution_time": 155.67875576019287, + "token_usage": { + "input_tokens": 340027, + "output_tokens": 5143, + "total_tokens": 345170, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 165.66, + "avg_input_tokens": 338621.2, + "avg_output_tokens": 5642.2, + "avg_total_tokens": 344263.5, + "avg_turn_count": 12.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 121.48146224021912, + "agent_execution_time": 121.48146224021912, + "token_usage": { + "input_tokens": 196331, + "output_tokens": 2849, + "total_tokens": 199180, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 99.45385217666626, + "agent_execution_time": 99.45385217666626, + "token_usage": { + "input_tokens": 160164, + "output_tokens": 2515, + "total_tokens": 162679, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 108.51680445671082, + "agent_execution_time": 108.51680445671082, + "token_usage": { + "input_tokens": 159280, + "output_tokens": 2589, + "total_tokens": 161869, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 122.8479917049408, + "agent_execution_time": 122.8479917049408, + "token_usage": { + "input_tokens": 343313, + "output_tokens": 4147, + "total_tokens": 347460, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 113.08, + "avg_input_tokens": 214772.0, + "avg_output_tokens": 3025.0, + "avg_total_tokens": 217797.0, + "avg_turn_count": 14.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 222.64552283287048, + "agent_execution_time": 222.64552283287048, + "token_usage": { + "input_tokens": 333265, + "output_tokens": 2954, + "total_tokens": 336219, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 12.07874083518982, + "agent_execution_time": 12.07874083518982, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 297.49602937698364, + "agent_execution_time": 297.49602937698364, + "token_usage": { + "input_tokens": 324974, + "output_tokens": 2858, + "total_tokens": 327832, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 143.3657112121582, + "agent_execution_time": 143.3657112121582, + "token_usage": { + "input_tokens": 56346, + "output_tokens": 3310, + "total_tokens": 59656, + "reasoning_tokens": 0 + }, + "turn_count": 2 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 168.9, + "avg_input_tokens": 178646.2, + "avg_output_tokens": 2280.5, + "avg_total_tokens": 180926.8, + "avg_turn_count": 7.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 91.16628623008728, + "agent_execution_time": 91.16628623008728, + "token_usage": { + "input_tokens": 445411, + "output_tokens": 6096, + "total_tokens": 451507, + "reasoning_tokens": 3738 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 69.33674907684326, + "agent_execution_time": 69.33674907684326, + "token_usage": { + "input_tokens": 594909, + "output_tokens": 4877, + "total_tokens": 599786, + "reasoning_tokens": 3163 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 185.30562829971313, + "agent_execution_time": 185.30562829971313, + "token_usage": { + "input_tokens": 247766, + "output_tokens": 16727, + "total_tokens": 264493, + "reasoning_tokens": 4748 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 149.45539236068726, + "agent_execution_time": 149.45539236068726, + "token_usage": { + "input_tokens": 586078, + "output_tokens": 9698, + "total_tokens": 595776, + "reasoning_tokens": 6097 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 123.82, + "avg_input_tokens": 468541.0, + "avg_output_tokens": 9349.5, + "avg_total_tokens": 477890.5, + "avg_turn_count": 20.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/harmony__multi_branch_commit_aggregation.json b/task_results/harmony__multi_branch_commit_aggregation.json index d0b7009c..392a7dac 100644 --- a/task_results/harmony__multi_branch_commit_aggregation.json +++ b/task_results/harmony__multi_branch_commit_aggregation.json @@ -2,6 +2,73 @@ "task": "harmony__multi_branch_commit_aggregation", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 233.56022357940674, + "agent_execution_time": 233.56022357940674, + "token_usage": { + "input_tokens": 309211, + "output_tokens": 3501, + "total_tokens": 312712, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1701.85706615448, + "agent_execution_time": 1701.85706615448, + "token_usage": { + "input_tokens": 268995, + "output_tokens": 37542, + "total_tokens": 306537, + "reasoning_tokens": 35439 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1543.3553655147552, + "agent_execution_time": 1543.3553655147552, + "token_usage": { + "input_tokens": 263603, + "output_tokens": 36653, + "total_tokens": 300256, + "reasoning_tokens": 34586 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1226.5491137504578, + "agent_execution_time": 1226.5491137504578, + "token_usage": { + "input_tokens": 179145, + "output_tokens": 35408, + "total_tokens": 214553, + "reasoning_tokens": 31524 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1176.33, + "avg_input_tokens": 255238.5, + "avg_output_tokens": 28276.0, + "avg_total_tokens": 283514.5, + "avg_turn_count": 7.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 597.5980651378632, - "agent_execution_time": 597.5980651378632, - "token_usage": { - "input_tokens": 1328632, - "output_tokens": 7930, - "total_tokens": 1336562, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 597.6, - "avg_input_tokens": 1328632.0, - "avg_output_tokens": 7930.0, - "avg_total_tokens": 1336562.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 146.04537844657898, + "agent_execution_time": 146.04537844657898, + "token_usage": { + "input_tokens": 372609, + "output_tokens": 5671, + "total_tokens": 378280, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 96.12090182304382, + "agent_execution_time": 96.12090182304382, + "token_usage": { + "input_tokens": 202555, + "output_tokens": 4053, + "total_tokens": 206608, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 136.93605017662048, + "agent_execution_time": 136.93605017662048, + "token_usage": { + "input_tokens": 356151, + "output_tokens": 5470, + "total_tokens": 361621, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 166.02852249145508, + "agent_execution_time": 166.02852249145508, + "token_usage": { + "input_tokens": 384563, + "output_tokens": 6307, + "total_tokens": 390870, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 136.28, + "avg_input_tokens": 328969.5, + "avg_output_tokens": 5375.2, + "avg_total_tokens": 334344.8, + "avg_turn_count": 7.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 336.24945735931396, + "agent_execution_time": 336.24945735931396, + "token_usage": { + "input_tokens": 427932, + "output_tokens": 6342, + "total_tokens": 434274, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 121.93927550315857, + "agent_execution_time": 121.93927550315857, + "token_usage": { + "input_tokens": 491377, + "output_tokens": 3281, + "total_tokens": 494658, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 173.14394235610962, + "agent_execution_time": 173.14394235610962, + "token_usage": { + "input_tokens": 698106, + "output_tokens": 3668, + "total_tokens": 701774, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-4", + "success": false, + "execution_time": 161.5775179862976, + "agent_execution_time": 161.5775179862976, + "token_usage": { + "input_tokens": 392176, + "output_tokens": 6267, + "total_tokens": 398443, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 198.23, + "avg_input_tokens": 502397.8, + "avg_output_tokens": 4889.5, + "avg_total_tokens": 507287.2, + "avg_turn_count": 22.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 285.6013243198395, + "agent_execution_time": 285.6013243198395, + "token_usage": { + "input_tokens": 655275, + "output_tokens": 3313, + "total_tokens": 658588, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.802415370941162, + "agent_execution_time": 10.802415370941162, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 297.0952076911926, + "agent_execution_time": 297.0952076911926, + "token_usage": { + "input_tokens": 286255, + "output_tokens": 3605, + "total_tokens": 289860, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 184.97724151611328, + "agent_execution_time": 184.97724151611328, + "token_usage": { + "input_tokens": 249974, + "output_tokens": 3205, + "total_tokens": 253179, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 194.62, + "avg_input_tokens": 297876.0, + "avg_output_tokens": 2530.8, + "avg_total_tokens": 300406.8, + "avg_turn_count": 9.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 105.11098074913025, + "agent_execution_time": 105.11098074913025, + "token_usage": { + "input_tokens": 345988, + "output_tokens": 10157, + "total_tokens": 356145, + "reasoning_tokens": 6669 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 84.2107036113739, + "agent_execution_time": 84.2107036113739, + "token_usage": { + "input_tokens": 535977, + "output_tokens": 7529, + "total_tokens": 543506, + "reasoning_tokens": 4897 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 85.45590472221375, + "agent_execution_time": 85.45590472221375, + "token_usage": { + "input_tokens": 180472, + "output_tokens": 9162, + "total_tokens": 189634, + "reasoning_tokens": 6482 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 78.71429514884949, + "agent_execution_time": 78.71429514884949, + "token_usage": { + "input_tokens": 215336, + "output_tokens": 8242, + "total_tokens": 223578, + "reasoning_tokens": 5688 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 88.37, + "avg_input_tokens": 319443.2, + "avg_output_tokens": 8772.5, + "avg_total_tokens": 328215.8, + "avg_turn_count": 7.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/harmony__release_management_workflow.json b/task_results/harmony__release_management_workflow.json index 140cccc7..73d6f3f8 100644 --- a/task_results/harmony__release_management_workflow.json +++ b/task_results/harmony__release_management_workflow.json @@ -2,6 +2,73 @@ "task": "harmony__release_management_workflow", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 407.90374398231506, + "agent_execution_time": 407.90374398231506, + "token_usage": { + "input_tokens": 1602791, + "output_tokens": 4474, + "total_tokens": 1607265, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1450.204171180725, + "agent_execution_time": 1450.204171180725, + "token_usage": { + "input_tokens": 503960, + "output_tokens": 28106, + "total_tokens": 532066, + "reasoning_tokens": 26182 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 2195.718644618988, + "agent_execution_time": 2195.718644618988, + "token_usage": { + "input_tokens": 550492, + "output_tokens": 47249, + "total_tokens": 597741, + "reasoning_tokens": 39114 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1330.544644355774, + "agent_execution_time": 1330.544644355774, + "token_usage": { + "input_tokens": 915634, + "output_tokens": 34559, + "total_tokens": 950193, + "reasoning_tokens": 32835 + }, + "turn_count": 16 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1346.09, + "avg_input_tokens": 893219.2, + "avg_output_tokens": 28597.0, + "avg_total_tokens": 921816.2, + "avg_turn_count": 16.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 565.036545753479, - "agent_execution_time": 565.036545753479, - "token_usage": { - "input_tokens": 1036903, - "output_tokens": 8321, - "total_tokens": 1045224, - "reasoning_tokens": 0 - }, - "turn_count": 17 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 565.04, - "avg_input_tokens": 1036903.0, - "avg_output_tokens": 8321.0, - "avg_total_tokens": 1045224.0, - "avg_turn_count": 17.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 204.74576830863953, + "agent_execution_time": 204.74576830863953, + "token_usage": { + "input_tokens": 1099725, + "output_tokens": 9419, + "total_tokens": 1109144, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 242.64568901062012, + "agent_execution_time": 242.64568901062012, + "token_usage": { + "input_tokens": 1669027, + "output_tokens": 10407, + "total_tokens": 1679434, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 263.83993458747864, + "agent_execution_time": 263.83993458747864, + "token_usage": { + "input_tokens": 1351392, + "output_tokens": 9899, + "total_tokens": 1361291, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 250.6093246936798, + "agent_execution_time": 250.6093246936798, + "token_usage": { + "input_tokens": 1382339, + "output_tokens": 11189, + "total_tokens": 1393528, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 240.46, + "avg_input_tokens": 1375620.8, + "avg_output_tokens": 10228.5, + "avg_total_tokens": 1385849.2, + "avg_turn_count": 19.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 694.2363014221191, + "agent_execution_time": 694.2363014221191, + "token_usage": { + "input_tokens": 2784644, + "output_tokens": 15661, + "total_tokens": 2800305, + "reasoning_tokens": 0 + }, + "turn_count": 55 + }, + { + "run": "run-2", + "success": false, + "execution_time": 789.9254982471466, + "agent_execution_time": 789.9254982471466, + "token_usage": { + "input_tokens": 5021113, + "output_tokens": 16355, + "total_tokens": 5037468, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-3", + "success": false, + "execution_time": 483.4850172996521, + "agent_execution_time": 483.4850172996521, + "token_usage": { + "input_tokens": 2777271, + "output_tokens": 11096, + "total_tokens": 2788367, + "reasoning_tokens": 0 + }, + "turn_count": 54 + }, + { + "run": "run-4", + "success": false, + "execution_time": 264.57943320274353, + "agent_execution_time": 264.57943320274353, + "token_usage": { + "input_tokens": 939261, + "output_tokens": 10632, + "total_tokens": 949893, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 558.06, + "avg_input_tokens": 2880572.2, + "avg_output_tokens": 13436.0, + "avg_total_tokens": 2894008.2, + "avg_turn_count": 57.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 614.7089338302612, + "agent_execution_time": 614.7089338302612, + "token_usage": { + "input_tokens": 1503438, + "output_tokens": 10493, + "total_tokens": 1513931, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": false, + "execution_time": 9.933528900146484, + "agent_execution_time": 9.933528900146484, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 730.019784450531, + "agent_execution_time": 730.019784450531, + "token_usage": { + "input_tokens": 1302166, + "output_tokens": 2883, + "total_tokens": 1305049, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 311.3378858566284, + "agent_execution_time": 311.3378858566284, + "token_usage": { + "input_tokens": 331360, + "output_tokens": 5289, + "total_tokens": 336649, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 416.5, + "avg_input_tokens": 784241.0, + "avg_output_tokens": 4666.2, + "avg_total_tokens": 788907.2, + "avg_turn_count": 13.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 139.07860112190247, + "agent_execution_time": 139.07860112190247, + "token_usage": { + "input_tokens": 1482872, + "output_tokens": 9260, + "total_tokens": 1492132, + "reasoning_tokens": 5998 + }, + "turn_count": 32 + }, + { + "run": "run-2", + "success": false, + "execution_time": 121.97672057151794, + "agent_execution_time": 121.97672057151794, + "token_usage": { + "input_tokens": 1221456, + "output_tokens": 8736, + "total_tokens": 1230192, + "reasoning_tokens": 6201 + }, + "turn_count": 27 + }, + { + "run": "run-3", + "success": false, + "execution_time": 50.831056118011475, + "agent_execution_time": 50.831056118011475, + "token_usage": { + "input_tokens": 617227, + "output_tokens": 3417, + "total_tokens": 620644, + "reasoning_tokens": 1978 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 97.53643751144409, + "agent_execution_time": 97.53643751144409, + "token_usage": { + "input_tokens": 737833, + "output_tokens": 7788, + "total_tokens": 745621, + "reasoning_tokens": 4724 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 102.36, + "avg_input_tokens": 1014847.0, + "avg_output_tokens": 7300.2, + "avg_total_tokens": 1022147.2, + "avg_turn_count": 22.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/it_trouble_shooting_hub__asset_retirement_migration.json b/task_results/it_trouble_shooting_hub__asset_retirement_migration.json index dfc0b826..3c288dec 100644 --- a/task_results/it_trouble_shooting_hub__asset_retirement_migration.json +++ b/task_results/it_trouble_shooting_hub__asset_retirement_migration.json @@ -2,6 +2,73 @@ "task": "it_trouble_shooting_hub__asset_retirement_migration", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 304.82218194007874, + "agent_execution_time": 304.82218194007874, + "token_usage": { + "input_tokens": 958294, + "output_tokens": 3066, + "total_tokens": 961360, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": false, + "execution_time": 333.83636116981506, + "agent_execution_time": 333.83636116981506, + "token_usage": { + "input_tokens": 6836, + "output_tokens": 8372, + "total_tokens": 15208, + "reasoning_tokens": 8310 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": false, + "execution_time": 177.39050602912903, + "agent_execution_time": 177.39050602912903, + "token_usage": { + "input_tokens": 355670, + "output_tokens": 1783, + "total_tokens": 357453, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1051.4181864261627, + "agent_execution_time": 1051.4181864261627, + "token_usage": { + "input_tokens": 30831, + "output_tokens": 9553, + "total_tokens": 40384, + "reasoning_tokens": 9388 + }, + "turn_count": 2 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 466.87, + "avg_input_tokens": 337907.8, + "avg_output_tokens": 5693.5, + "avg_total_tokens": 343601.2, + "avg_turn_count": 10.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 336.1930365562439, - "agent_execution_time": 336.1930365562439, - "token_usage": { - "input_tokens": 511670, - "output_tokens": 3064, - "total_tokens": 514734, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 336.19, - "avg_input_tokens": 511670.0, - "avg_output_tokens": 3064.0, - "avg_total_tokens": 514734.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 117.95915627479553, + "agent_execution_time": 117.95915627479553, + "token_usage": { + "input_tokens": 333402, + "output_tokens": 3060, + "total_tokens": 336462, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 239.47261881828308, + "agent_execution_time": 239.47261881828308, + "token_usage": { + "input_tokens": 2312265, + "output_tokens": 6560, + "total_tokens": 2318825, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": true, + "execution_time": 185.89870953559875, + "agent_execution_time": 185.89870953559875, + "token_usage": { + "input_tokens": 1846951, + "output_tokens": 4330, + "total_tokens": 1851281, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 223.02784872055054, + "agent_execution_time": 223.02784872055054, + "token_usage": { + "input_tokens": 1209119, + "output_tokens": 5113, + "total_tokens": 1214232, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 191.59, + "avg_input_tokens": 1425434.2, + "avg_output_tokens": 4765.8, + "avg_total_tokens": 1430200.0, + "avg_turn_count": 20.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 97.5825834274292, + "agent_execution_time": 97.5825834274292, + "token_usage": { + "input_tokens": 319635, + "output_tokens": 2674, + "total_tokens": 322309, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": true, + "execution_time": 137.63185620307922, + "agent_execution_time": 137.63185620307922, + "token_usage": { + "input_tokens": 594531, + "output_tokens": 1876, + "total_tokens": 596407, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": true, + "execution_time": 142.25383472442627, + "agent_execution_time": 142.25383472442627, + "token_usage": { + "input_tokens": 358252, + "output_tokens": 2753, + "total_tokens": 361005, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": true, + "execution_time": 135.81972861289978, + "agent_execution_time": 135.81972861289978, + "token_usage": { + "input_tokens": 462541, + "output_tokens": 3062, + "total_tokens": 465603, + "reasoning_tokens": 0 + }, + "turn_count": 16 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 128.32, + "avg_input_tokens": 433739.8, + "avg_output_tokens": 2591.2, + "avg_total_tokens": 436331.0, + "avg_turn_count": 15.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 371.3148522377014, + "agent_execution_time": 371.3148522377014, + "token_usage": { + "input_tokens": 1174379, + "output_tokens": 4356, + "total_tokens": 1178735, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-2", + "success": true, + "execution_time": 273.0747570991516, + "agent_execution_time": 273.0747570991516, + "token_usage": { + "input_tokens": 514753, + "output_tokens": 2589, + "total_tokens": 517342, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": true, + "execution_time": 208.73402976989746, + "agent_execution_time": 208.73402976989746, + "token_usage": { + "input_tokens": 539875, + "output_tokens": 2028, + "total_tokens": 541903, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": true, + "execution_time": 188.8012306690216, + "agent_execution_time": 188.8012306690216, + "token_usage": { + "input_tokens": 497125, + "output_tokens": 2047, + "total_tokens": 499172, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 260.48, + "avg_input_tokens": 681533.0, + "avg_output_tokens": 2755.0, + "avg_total_tokens": 684288.0, + "avg_turn_count": 19.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 87.78935718536377, + "agent_execution_time": 87.78935718536377, + "token_usage": { + "input_tokens": 277371, + "output_tokens": 6430, + "total_tokens": 283801, + "reasoning_tokens": 4535 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 49.56605386734009, + "agent_execution_time": 49.56605386734009, + "token_usage": { + "input_tokens": 187634, + "output_tokens": 5542, + "total_tokens": 193176, + "reasoning_tokens": 4157 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 152.46633458137512, + "agent_execution_time": 152.46633458137512, + "token_usage": { + "input_tokens": 902256, + "output_tokens": 11601, + "total_tokens": 913857, + "reasoning_tokens": 7763 + }, + "turn_count": 45 + }, + { + "run": "run-4", + "success": false, + "execution_time": 91.61820769309998, + "agent_execution_time": 91.61820769309998, + "token_usage": { + "input_tokens": 507113, + "output_tokens": 8418, + "total_tokens": 515531, + "reasoning_tokens": 6454 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 95.36, + "avg_input_tokens": 468593.5, + "avg_output_tokens": 7997.8, + "avg_total_tokens": 476591.2, + "avg_turn_count": 25.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/it_trouble_shooting_hub__security_audit_ticket.json b/task_results/it_trouble_shooting_hub__security_audit_ticket.json index 4c19b3a9..8d5d75fa 100644 --- a/task_results/it_trouble_shooting_hub__security_audit_ticket.json +++ b/task_results/it_trouble_shooting_hub__security_audit_ticket.json @@ -2,6 +2,73 @@ "task": "it_trouble_shooting_hub__security_audit_ticket", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 222.0452995300293, + "agent_execution_time": 222.0452995300293, + "token_usage": { + "input_tokens": 536938, + "output_tokens": 2505, + "total_tokens": 539443, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": true, + "execution_time": 1014.8265445232391, + "agent_execution_time": 1014.8265445232391, + "token_usage": { + "input_tokens": 209350, + "output_tokens": 21222, + "total_tokens": 230572, + "reasoning_tokens": 20029 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1007.6195752620697, + "agent_execution_time": 1007.6195752620697, + "token_usage": { + "input_tokens": 4683674, + "output_tokens": 9331, + "total_tokens": 4693005, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-4", + "success": true, + "execution_time": 990.9929640293121, + "agent_execution_time": 990.9929640293121, + "token_usage": { + "input_tokens": 288830, + "output_tokens": 26772, + "total_tokens": 315602, + "reasoning_tokens": 25457 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 808.87, + "avg_input_tokens": 1429698.0, + "avg_output_tokens": 14957.5, + "avg_total_tokens": 1444655.5, + "avg_turn_count": 32.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 185.3166160583496, - "agent_execution_time": 185.3166160583496, - "token_usage": { - "input_tokens": 215873, - "output_tokens": 2016, - "total_tokens": 217889, - "reasoning_tokens": null - }, - "turn_count": 8 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 185.32, - "avg_input_tokens": 215873.0, - "avg_output_tokens": 2016.0, - "avg_total_tokens": 217889.0, - "avg_turn_count": 8.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 145.1133677959442, + "agent_execution_time": 145.1133677959442, + "token_usage": { + "input_tokens": 418782, + "output_tokens": 2625, + "total_tokens": 421407, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 249.26999831199646, + "agent_execution_time": 249.26999831199646, + "token_usage": { + "input_tokens": 1011424, + "output_tokens": 7724, + "total_tokens": 1019148, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": true, + "execution_time": 83.56927943229675, + "agent_execution_time": 83.56927943229675, + "token_usage": { + "input_tokens": 402139, + "output_tokens": 2662, + "total_tokens": 404801, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": true, + "execution_time": 191.2257583141327, + "agent_execution_time": 191.2257583141327, + "token_usage": { + "input_tokens": 811191, + "output_tokens": 4054, + "total_tokens": 815245, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 167.29, + "avg_input_tokens": 660884.0, + "avg_output_tokens": 4266.2, + "avg_total_tokens": 665150.2, + "avg_turn_count": 15.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 174.7085645198822, + "agent_execution_time": 174.7085645198822, + "token_usage": { + "input_tokens": 353844, + "output_tokens": 4284, + "total_tokens": 358128, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": true, + "execution_time": 60.417407512664795, + "agent_execution_time": 60.417407512664795, + "token_usage": { + "input_tokens": 177608, + "output_tokens": 1083, + "total_tokens": 178691, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": true, + "execution_time": 212.88041400909424, + "agent_execution_time": 212.88041400909424, + "token_usage": { + "input_tokens": 348894, + "output_tokens": 4805, + "total_tokens": 353699, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": true, + "execution_time": 189.17696619033813, + "agent_execution_time": 189.17696619033813, + "token_usage": { + "input_tokens": 836349, + "output_tokens": 4582, + "total_tokens": 840931, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 159.3, + "avg_input_tokens": 429173.8, + "avg_output_tokens": 3688.5, + "avg_total_tokens": 432862.2, + "avg_turn_count": 14.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 196.57794880867004, + "agent_execution_time": 196.57794880867004, + "token_usage": { + "input_tokens": 635734, + "output_tokens": 2115, + "total_tokens": 637849, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": true, + "execution_time": 225.43587398529053, + "agent_execution_time": 225.43587398529053, + "token_usage": { + "input_tokens": 438608, + "output_tokens": 2510, + "total_tokens": 441118, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 318.1488287448883, + "agent_execution_time": 318.1488287448883, + "token_usage": { + "input_tokens": 1840410, + "output_tokens": 2585, + "total_tokens": 1842995, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": true, + "execution_time": 186.78525686264038, + "agent_execution_time": 186.78525686264038, + "token_usage": { + "input_tokens": 527937, + "output_tokens": 1913, + "total_tokens": 529850, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 231.74, + "avg_input_tokens": 860672.2, + "avg_output_tokens": 2280.8, + "avg_total_tokens": 862953.0, + "avg_turn_count": 16.25, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 73.72738265991211, + "agent_execution_time": 73.72738265991211, + "token_usage": { + "input_tokens": 298650, + "output_tokens": 6796, + "total_tokens": 305446, + "reasoning_tokens": 5373 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 63.83225989341736, + "agent_execution_time": 63.83225989341736, + "token_usage": { + "input_tokens": 262008, + "output_tokens": 5468, + "total_tokens": 267476, + "reasoning_tokens": 4027 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 47.03505611419678, + "agent_execution_time": 47.03505611419678, + "token_usage": { + "input_tokens": 134029, + "output_tokens": 4678, + "total_tokens": 138707, + "reasoning_tokens": 3113 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 66.14121913909912, + "agent_execution_time": 66.14121913909912, + "token_usage": { + "input_tokens": 167096, + "output_tokens": 4613, + "total_tokens": 171709, + "reasoning_tokens": 3549 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 62.68, + "avg_input_tokens": 215445.8, + "avg_output_tokens": 5388.8, + "avg_total_tokens": 220834.5, + "avg_turn_count": 12.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/it_trouble_shooting_hub__verification_expired_update.json b/task_results/it_trouble_shooting_hub__verification_expired_update.json index 535e2d5d..50eb04ca 100644 --- a/task_results/it_trouble_shooting_hub__verification_expired_update.json +++ b/task_results/it_trouble_shooting_hub__verification_expired_update.json @@ -2,6 +2,73 @@ "task": "it_trouble_shooting_hub__verification_expired_update", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 212.68869137763977, + "agent_execution_time": 212.68869137763977, + "token_usage": { + "input_tokens": 461583, + "output_tokens": 2347, + "total_tokens": 463930, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 2508.9891328811646, + "agent_execution_time": 2508.9891328811646, + "token_usage": { + "input_tokens": 390069, + "output_tokens": 51477, + "total_tokens": 441546, + "reasoning_tokens": 49945 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 326.9920198917389, + "agent_execution_time": 326.9920198917389, + "token_usage": { + "input_tokens": 1327503, + "output_tokens": 2938, + "total_tokens": 1330441, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": false, + "execution_time": 398.0065813064575, + "agent_execution_time": 398.0065813064575, + "token_usage": { + "input_tokens": 6688, + "output_tokens": 12021, + "total_tokens": 18709, + "reasoning_tokens": 11959 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 861.67, + "avg_input_tokens": 546460.8, + "avg_output_tokens": 17195.8, + "avg_total_tokens": 563656.5, + "avg_turn_count": 13.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 284.5214612483978, - "agent_execution_time": 284.5214612483978, - "token_usage": { - "input_tokens": 991980, - "output_tokens": 3993, - "total_tokens": 995973, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 284.52, - "avg_input_tokens": 991980.0, - "avg_output_tokens": 3993.0, - "avg_total_tokens": 995973.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 366.1456768512726, + "agent_execution_time": 366.1456768512726, + "token_usage": { + "input_tokens": 1450904, + "output_tokens": 8911, + "total_tokens": 1459815, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-2", + "success": false, + "execution_time": 187.86872124671936, + "agent_execution_time": 187.86872124671936, + "token_usage": { + "input_tokens": 1292817, + "output_tokens": 5843, + "total_tokens": 1298660, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": false, + "execution_time": 250.73846244812012, + "agent_execution_time": 250.73846244812012, + "token_usage": { + "input_tokens": 1249920, + "output_tokens": 4355, + "total_tokens": 1254275, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 218.2809386253357, + "agent_execution_time": 218.2809386253357, + "token_usage": { + "input_tokens": 1035185, + "output_tokens": 5107, + "total_tokens": 1040292, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 255.76, + "avg_input_tokens": 1257206.5, + "avg_output_tokens": 6054.0, + "avg_total_tokens": 1263260.5, + "avg_turn_count": 23.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 56.17271971702576, + "agent_execution_time": 56.17271971702576, + "token_usage": { + "input_tokens": 203233, + "output_tokens": 1619, + "total_tokens": 204852, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 165.1204538345337, + "agent_execution_time": 165.1204538345337, + "token_usage": { + "input_tokens": 461426, + "output_tokens": 4362, + "total_tokens": 465788, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-3", + "success": false, + "execution_time": 175.70588207244873, + "agent_execution_time": 175.70588207244873, + "token_usage": { + "input_tokens": 304112, + "output_tokens": 3050, + "total_tokens": 307162, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": false, + "execution_time": 264.26389479637146, + "agent_execution_time": 264.26389479637146, + "token_usage": { + "input_tokens": 749007, + "output_tokens": 5457, + "total_tokens": 754464, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 165.32, + "avg_input_tokens": 429444.5, + "avg_output_tokens": 3622.0, + "avg_total_tokens": 433066.5, + "avg_turn_count": 16.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 229.99783205986023, + "agent_execution_time": 229.99783205986023, + "token_usage": { + "input_tokens": 766405, + "output_tokens": 2391, + "total_tokens": 768796, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 140.7087812423706, + "agent_execution_time": 140.7087812423706, + "token_usage": { + "input_tokens": 301093, + "output_tokens": 1276, + "total_tokens": 302369, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 203.37706303596497, + "agent_execution_time": 203.37706303596497, + "token_usage": { + "input_tokens": 625154, + "output_tokens": 2019, + "total_tokens": 627173, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 160.08109521865845, + "agent_execution_time": 160.08109521865845, + "token_usage": { + "input_tokens": 460592, + "output_tokens": 1725, + "total_tokens": 462317, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 183.54, + "avg_input_tokens": 538311.0, + "avg_output_tokens": 1852.8, + "avg_total_tokens": 540163.8, + "avg_turn_count": 14.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 91.70930600166321, + "agent_execution_time": 91.70930600166321, + "token_usage": { + "input_tokens": 385358, + "output_tokens": 9969, + "total_tokens": 395327, + "reasoning_tokens": 8523 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 80.19039297103882, + "agent_execution_time": 80.19039297103882, + "token_usage": { + "input_tokens": 450985, + "output_tokens": 6961, + "total_tokens": 457946, + "reasoning_tokens": 5075 + }, + "turn_count": 27 + }, + { + "run": "run-3", + "success": false, + "execution_time": 109.55777955055237, + "agent_execution_time": 109.55777955055237, + "token_usage": { + "input_tokens": 632648, + "output_tokens": 7090, + "total_tokens": 639738, + "reasoning_tokens": 5443 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": false, + "execution_time": 144.57292103767395, + "agent_execution_time": 144.57292103767395, + "token_usage": { + "input_tokens": 774930, + "output_tokens": 10179, + "total_tokens": 785109, + "reasoning_tokens": 7214 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 106.51, + "avg_input_tokens": 560980.2, + "avg_output_tokens": 8549.8, + "avg_total_tokens": 569530.0, + "avg_turn_count": 23.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/japan_travel_planner__daily_itinerary_overview.json b/task_results/japan_travel_planner__daily_itinerary_overview.json index 911ddf84..b706242b 100644 --- a/task_results/japan_travel_planner__daily_itinerary_overview.json +++ b/task_results/japan_travel_planner__daily_itinerary_overview.json @@ -2,6 +2,73 @@ "task": "japan_travel_planner__daily_itinerary_overview", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 247.0055320262909, + "agent_execution_time": 247.0055320262909, + "token_usage": { + "input_tokens": 682445, + "output_tokens": 3097, + "total_tokens": 685542, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1449.372210264206, + "agent_execution_time": 1449.372210264206, + "token_usage": { + "input_tokens": 151449, + "output_tokens": 35684, + "total_tokens": 187133, + "reasoning_tokens": 30556 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 526.1702470779419, + "agent_execution_time": 526.1702470779419, + "token_usage": { + "input_tokens": 541614, + "output_tokens": 2567, + "total_tokens": 544181, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 929.0850973129272, + "agent_execution_time": 929.0850973129272, + "token_usage": { + "input_tokens": 102091, + "output_tokens": 26858, + "total_tokens": 128949, + "reasoning_tokens": 24100 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 787.91, + "avg_input_tokens": 369399.8, + "avg_output_tokens": 17051.5, + "avg_total_tokens": 386451.2, + "avg_turn_count": 7.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 127.18816041946411, - "agent_execution_time": 127.18816041946411, - "token_usage": { - "input_tokens": 369809, - "output_tokens": 3078, - "total_tokens": 372887, - "reasoning_tokens": null - }, - "turn_count": 5 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 127.19, - "avg_input_tokens": 369809.0, - "avg_output_tokens": 3078.0, - "avg_total_tokens": 372887.0, - "avg_turn_count": 5.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 81.93832159042358, + "agent_execution_time": 81.93832159042358, + "token_usage": { + "input_tokens": 281329, + "output_tokens": 4319, + "total_tokens": 285648, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 73.66770553588867, + "agent_execution_time": 73.66770553588867, + "token_usage": { + "input_tokens": 269195, + "output_tokens": 4516, + "total_tokens": 273711, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 78.67584681510925, + "agent_execution_time": 78.67584681510925, + "token_usage": { + "input_tokens": 326669, + "output_tokens": 4199, + "total_tokens": 330868, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 73.6284589767456, + "agent_execution_time": 73.6284589767456, + "token_usage": { + "input_tokens": 283316, + "output_tokens": 4912, + "total_tokens": 288228, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 76.98, + "avg_input_tokens": 290127.2, + "avg_output_tokens": 4486.5, + "avg_total_tokens": 294613.8, + "avg_turn_count": 5.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 158.95628833770752, + "agent_execution_time": 158.95628833770752, + "token_usage": { + "input_tokens": 316928, + "output_tokens": 3311, + "total_tokens": 320239, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 139.56335735321045, + "agent_execution_time": 139.56335735321045, + "token_usage": { + "input_tokens": 244789, + "output_tokens": 3458, + "total_tokens": 248247, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 121.52990245819092, + "agent_execution_time": 121.52990245819092, + "token_usage": { + "input_tokens": 146213, + "output_tokens": 3661, + "total_tokens": 149874, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 185.03372359275818, + "agent_execution_time": 185.03372359275818, + "token_usage": { + "input_tokens": 159223, + "output_tokens": 3820, + "total_tokens": 163043, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 151.27, + "avg_input_tokens": 216788.2, + "avg_output_tokens": 3562.5, + "avg_total_tokens": 220350.8, + "avg_turn_count": 7.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 246.66852021217346, + "agent_execution_time": 246.66852021217346, + "token_usage": { + "input_tokens": 771114, + "output_tokens": 2888, + "total_tokens": 774002, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 99.9729573726654, + "agent_execution_time": 99.9729573726654, + "token_usage": { + "input_tokens": 105094, + "output_tokens": 1717, + "total_tokens": 106811, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 400.91640996932983, + "agent_execution_time": 400.91640996932983, + "token_usage": { + "input_tokens": 1124434, + "output_tokens": 4515, + "total_tokens": 1128949, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 488.61013412475586, + "agent_execution_time": 488.61013412475586, + "token_usage": { + "input_tokens": 2441300, + "output_tokens": 4978, + "total_tokens": 2446278, + "reasoning_tokens": 0 + }, + "turn_count": 31 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 309.04, + "avg_input_tokens": 1110485.5, + "avg_output_tokens": 3524.5, + "avg_total_tokens": 1114010.0, + "avg_turn_count": 15.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 46.992114782333374, + "agent_execution_time": 46.992114782333374, + "token_usage": { + "input_tokens": 319155, + "output_tokens": 3785, + "total_tokens": 322940, + "reasoning_tokens": 3087 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 373.01144433021545, + "agent_execution_time": 373.01144433021545, + "token_usage": { + "input_tokens": 144028, + "output_tokens": 71698, + "total_tokens": 215726, + "reasoning_tokens": 2743 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 47.8009614944458, + "agent_execution_time": 47.8009614944458, + "token_usage": { + "input_tokens": 219979, + "output_tokens": 6032, + "total_tokens": 226011, + "reasoning_tokens": 4680 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 123.42545127868652, + "agent_execution_time": 123.42545127868652, + "token_usage": { + "input_tokens": 912289, + "output_tokens": 13390, + "total_tokens": 925679, + "reasoning_tokens": 5512 + }, + "turn_count": 16 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 147.81, + "avg_input_tokens": 398862.8, + "avg_output_tokens": 23726.2, + "avg_total_tokens": 422589.0, + "avg_turn_count": 8.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/japan_travel_planner__packing_progress_summary.json b/task_results/japan_travel_planner__packing_progress_summary.json index af2ee715..a7376df8 100644 --- a/task_results/japan_travel_planner__packing_progress_summary.json +++ b/task_results/japan_travel_planner__packing_progress_summary.json @@ -2,6 +2,73 @@ "task": "japan_travel_planner__packing_progress_summary", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 296.34033703804016, + "agent_execution_time": 296.34033703804016, + "token_usage": { + "input_tokens": 1145499, + "output_tokens": 1985, + "total_tokens": 1147484, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1949.6066291332245, + "agent_execution_time": 1949.6066291332245, + "token_usage": { + "input_tokens": 372929, + "output_tokens": 40385, + "total_tokens": 413314, + "reasoning_tokens": 38253 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 254.55807900428772, + "agent_execution_time": 254.55807900428772, + "token_usage": { + "input_tokens": 1012181, + "output_tokens": 1916, + "total_tokens": 1014097, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1101.5449602603912, + "agent_execution_time": 1101.5449602603912, + "token_usage": { + "input_tokens": 256167, + "output_tokens": 28183, + "total_tokens": 284350, + "reasoning_tokens": 26361 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 900.51, + "avg_input_tokens": 696694.0, + "avg_output_tokens": 18117.2, + "avg_total_tokens": 714811.2, + "avg_turn_count": 14.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 277.22349977493286, - "agent_execution_time": 277.22349977493286, - "token_usage": { - "input_tokens": 1953987, - "output_tokens": 4197, - "total_tokens": 1958184, - "reasoning_tokens": null - }, - "turn_count": 21 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 277.22, - "avg_input_tokens": 1953987.0, - "avg_output_tokens": 4197.0, - "avg_total_tokens": 1958184.0, - "avg_turn_count": 21.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 116.83711194992065, + "agent_execution_time": 116.83711194992065, + "token_usage": { + "input_tokens": 436667, + "output_tokens": 3514, + "total_tokens": 440181, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 126.43457913398743, + "agent_execution_time": 126.43457913398743, + "token_usage": { + "input_tokens": 889598, + "output_tokens": 4321, + "total_tokens": 893919, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 117.8135359287262, + "agent_execution_time": 117.8135359287262, + "token_usage": { + "input_tokens": 483602, + "output_tokens": 3648, + "total_tokens": 487250, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 162.7522430419922, + "agent_execution_time": 162.7522430419922, + "token_usage": { + "input_tokens": 813250, + "output_tokens": 5227, + "total_tokens": 818477, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 130.96, + "avg_input_tokens": 655779.2, + "avg_output_tokens": 4177.5, + "avg_total_tokens": 659956.8, + "avg_turn_count": 11.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 252.8636691570282, + "agent_execution_time": 252.8636691570282, + "token_usage": { + "input_tokens": 1144685, + "output_tokens": 4263, + "total_tokens": 1148948, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": false, + "execution_time": 143.458890914917, + "agent_execution_time": 143.458890914917, + "token_usage": { + "input_tokens": 860752, + "output_tokens": 1771, + "total_tokens": 862523, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 224.67261695861816, + "agent_execution_time": 224.67261695861816, + "token_usage": { + "input_tokens": 877006, + "output_tokens": 4348, + "total_tokens": 881354, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": false, + "execution_time": 240.52384519577026, + "agent_execution_time": 240.52384519577026, + "token_usage": { + "input_tokens": 974516, + "output_tokens": 4239, + "total_tokens": 978755, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 215.38, + "avg_input_tokens": 964239.8, + "avg_output_tokens": 3655.2, + "avg_total_tokens": 967895.0, + "avg_turn_count": 22.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 313.274475812912, + "agent_execution_time": 313.274475812912, + "token_usage": { + "input_tokens": 1510864, + "output_tokens": 2414, + "total_tokens": 1513278, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 129.94551348686218, + "agent_execution_time": 129.94551348686218, + "token_usage": { + "input_tokens": 232836, + "output_tokens": 1535, + "total_tokens": 234371, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 292.5775797367096, + "agent_execution_time": 292.5775797367096, + "token_usage": { + "input_tokens": 1267998, + "output_tokens": 2074, + "total_tokens": 1270072, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": false, + "execution_time": 287.5447256565094, + "agent_execution_time": 287.5447256565094, + "token_usage": { + "input_tokens": 1323449, + "output_tokens": 2059, + "total_tokens": 1325508, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 255.84, + "avg_input_tokens": 1083786.8, + "avg_output_tokens": 2020.5, + "avg_total_tokens": 1085807.2, + "avg_turn_count": 20.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 115.71496367454529, + "agent_execution_time": 115.71496367454529, + "token_usage": { + "input_tokens": 556806, + "output_tokens": 9627, + "total_tokens": 566433, + "reasoning_tokens": 4918 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 175.28860759735107, + "agent_execution_time": 175.28860759735107, + "token_usage": { + "input_tokens": 353562, + "output_tokens": 18224, + "total_tokens": 371786, + "reasoning_tokens": 8794 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 454.695285320282, + "agent_execution_time": 454.695285320282, + "token_usage": { + "input_tokens": 13886094, + "output_tokens": 14226, + "total_tokens": 13900320, + "reasoning_tokens": 11503 + }, + "turn_count": 36 + }, + { + "run": "run-4", + "success": false, + "execution_time": 65.58109784126282, + "agent_execution_time": 65.58109784126282, + "token_usage": { + "input_tokens": 367626, + "output_tokens": 6645, + "total_tokens": 374271, + "reasoning_tokens": 2888 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 202.82, + "avg_input_tokens": 3791022.0, + "avg_output_tokens": 12180.5, + "avg_total_tokens": 3803202.5, + "avg_turn_count": 17.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/japan_travel_planner__remove_osaka_itinerary.json b/task_results/japan_travel_planner__remove_osaka_itinerary.json index 1d934b55..7e1ad83a 100644 --- a/task_results/japan_travel_planner__remove_osaka_itinerary.json +++ b/task_results/japan_travel_planner__remove_osaka_itinerary.json @@ -2,6 +2,73 @@ "task": "japan_travel_planner__remove_osaka_itinerary", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 78.14082646369934, + "agent_execution_time": 78.14082646369934, + "token_usage": { + "input_tokens": 136775, + "output_tokens": 881, + "total_tokens": 137656, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 378.673992395401, + "agent_execution_time": 378.673992395401, + "token_usage": { + "input_tokens": 75380, + "output_tokens": 8564, + "total_tokens": 83944, + "reasoning_tokens": 7959 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": true, + "execution_time": 128.21069312095642, + "agent_execution_time": 128.21069312095642, + "token_usage": { + "input_tokens": 472884, + "output_tokens": 943, + "total_tokens": 473827, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": true, + "execution_time": 385.25209307670593, + "agent_execution_time": 385.25209307670593, + "token_usage": { + "input_tokens": 79682, + "output_tokens": 9774, + "total_tokens": 89456, + "reasoning_tokens": 9112 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 242.57, + "avg_input_tokens": 191180.2, + "avg_output_tokens": 5040.5, + "avg_total_tokens": 196220.8, + "avg_turn_count": 5.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 94.12291431427002, - "agent_execution_time": 94.12291431427002, - "token_usage": { - "input_tokens": 792892, - "output_tokens": 1289, - "total_tokens": 794181, - "reasoning_tokens": null - }, - "turn_count": 8 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 94.12, - "avg_input_tokens": 792892.0, - "avg_output_tokens": 1289.0, - "avg_total_tokens": 794181.0, - "avg_turn_count": 8.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 44.59681296348572, + "agent_execution_time": 44.59681296348572, + "token_usage": { + "input_tokens": 201381, + "output_tokens": 1376, + "total_tokens": 202757, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": true, + "execution_time": 36.224655628204346, + "agent_execution_time": 36.224655628204346, + "token_usage": { + "input_tokens": 179392, + "output_tokens": 1099, + "total_tokens": 180491, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": true, + "execution_time": 180.99512124061584, + "agent_execution_time": 180.99512124061584, + "token_usage": { + "input_tokens": 470201, + "output_tokens": 1458, + "total_tokens": 471659, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 42.02324724197388, + "agent_execution_time": 42.02324724197388, + "token_usage": { + "input_tokens": 206275, + "output_tokens": 1094, + "total_tokens": 207369, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 75.96, + "avg_input_tokens": 264312.2, + "avg_output_tokens": 1256.8, + "avg_total_tokens": 265569.0, + "avg_turn_count": 5.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 61.98464107513428, + "agent_execution_time": 61.98464107513428, + "token_usage": { + "input_tokens": 121115, + "output_tokens": 1487, + "total_tokens": 122602, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 92.15318036079407, + "agent_execution_time": 92.15318036079407, + "token_usage": { + "input_tokens": 309254, + "output_tokens": 1103, + "total_tokens": 310357, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": true, + "execution_time": 68.0990526676178, + "agent_execution_time": 68.0990526676178, + "token_usage": { + "input_tokens": 199959, + "output_tokens": 2119, + "total_tokens": 202078, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": true, + "execution_time": 93.30764722824097, + "agent_execution_time": 93.30764722824097, + "token_usage": { + "input_tokens": 241781, + "output_tokens": 2180, + "total_tokens": 243961, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 78.89, + "avg_input_tokens": 218027.2, + "avg_output_tokens": 1722.2, + "avg_total_tokens": 219749.5, + "avg_turn_count": 11.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 90.00246691703796, + "agent_execution_time": 90.00246691703796, + "token_usage": { + "input_tokens": 137378, + "output_tokens": 975, + "total_tokens": 138353, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 173.94200658798218, + "agent_execution_time": 173.94200658798218, + "token_usage": { + "input_tokens": 345204, + "output_tokens": 803, + "total_tokens": 346007, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": true, + "execution_time": 130.4261281490326, + "agent_execution_time": 130.4261281490326, + "token_usage": { + "input_tokens": 431315, + "output_tokens": 1072, + "total_tokens": 432387, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": true, + "execution_time": 158.96713852882385, + "agent_execution_time": 158.96713852882385, + "token_usage": { + "input_tokens": 485797, + "output_tokens": 1347, + "total_tokens": 487144, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 138.33, + "avg_input_tokens": 349923.5, + "avg_output_tokens": 1049.2, + "avg_total_tokens": 350972.8, + "avg_turn_count": 8.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 26.992112398147583, + "agent_execution_time": 26.992112398147583, + "token_usage": { + "input_tokens": 53136, + "output_tokens": 1918, + "total_tokens": 55054, + "reasoning_tokens": 1549 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": true, + "execution_time": 28.564209699630737, + "agent_execution_time": 28.564209699630737, + "token_usage": { + "input_tokens": 104649, + "output_tokens": 2152, + "total_tokens": 106801, + "reasoning_tokens": 1563 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": true, + "execution_time": 42.16823625564575, + "agent_execution_time": 42.16823625564575, + "token_usage": { + "input_tokens": 188289, + "output_tokens": 2700, + "total_tokens": 190989, + "reasoning_tokens": 2231 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": true, + "execution_time": 24.776986122131348, + "agent_execution_time": 24.776986122131348, + "token_usage": { + "input_tokens": 67944, + "output_tokens": 1521, + "total_tokens": 69465, + "reasoning_tokens": 1077 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 30.63, + "avg_input_tokens": 103504.5, + "avg_output_tokens": 2072.8, + "avg_total_tokens": 105577.2, + "avg_turn_count": 6.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/japan_travel_planner__restaurant_expenses_sync.json b/task_results/japan_travel_planner__restaurant_expenses_sync.json index d82ab428..d6547cbc 100644 --- a/task_results/japan_travel_planner__restaurant_expenses_sync.json +++ b/task_results/japan_travel_planner__restaurant_expenses_sync.json @@ -2,6 +2,73 @@ "task": "japan_travel_planner__restaurant_expenses_sync", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 230.64241123199463, + "agent_execution_time": 230.64241123199463, + "token_usage": { + "input_tokens": 353236, + "output_tokens": 2874, + "total_tokens": 356110, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1191.5031871795654, + "agent_execution_time": 1191.5031871795654, + "token_usage": { + "input_tokens": 80571, + "output_tokens": 30284, + "total_tokens": 110855, + "reasoning_tokens": 28241 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 230.10843515396118, + "agent_execution_time": 230.10843515396118, + "token_usage": { + "input_tokens": 364540, + "output_tokens": 2657, + "total_tokens": 367197, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": true, + "execution_time": 1865.1096251010895, + "agent_execution_time": 1865.1096251010895, + "token_usage": { + "input_tokens": 197560, + "output_tokens": 45201, + "total_tokens": 242761, + "reasoning_tokens": 40564 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 879.34, + "avg_input_tokens": 248976.8, + "avg_output_tokens": 20254.0, + "avg_total_tokens": 269230.8, + "avg_turn_count": 14.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 125.08707022666931, - "agent_execution_time": 125.08707022666931, - "token_usage": { - "input_tokens": 255917, - "output_tokens": 2431, - "total_tokens": 258348, - "reasoning_tokens": null - }, - "turn_count": 10 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 125.09, - "avg_input_tokens": 255917.0, - "avg_output_tokens": 2431.0, - "avg_total_tokens": 258348.0, - "avg_turn_count": 10.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 111.48391270637512, + "agent_execution_time": 111.48391270637512, + "token_usage": { + "input_tokens": 286204, + "output_tokens": 3778, + "total_tokens": 289982, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 68.60682034492493, + "agent_execution_time": 68.60682034492493, + "token_usage": { + "input_tokens": 158082, + "output_tokens": 2042, + "total_tokens": 160124, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 114.06821250915527, + "agent_execution_time": 114.06821250915527, + "token_usage": { + "input_tokens": 296315, + "output_tokens": 4247, + "total_tokens": 300562, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": true, + "execution_time": 81.31473469734192, + "agent_execution_time": 81.31473469734192, + "token_usage": { + "input_tokens": 124759, + "output_tokens": 2757, + "total_tokens": 127516, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 93.87, + "avg_input_tokens": 216340.0, + "avg_output_tokens": 3206.0, + "avg_total_tokens": 219546.0, + "avg_turn_count": 9.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 121.64657688140869, + "agent_execution_time": 121.64657688140869, + "token_usage": { + "input_tokens": 134034, + "output_tokens": 2139, + "total_tokens": 136173, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 96.07570600509644, + "agent_execution_time": 96.07570600509644, + "token_usage": { + "input_tokens": 416756, + "output_tokens": 1440, + "total_tokens": 418196, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 72.52384734153748, + "agent_execution_time": 72.52384734153748, + "token_usage": { + "input_tokens": 273182, + "output_tokens": 2077, + "total_tokens": 275259, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 42.94132041931152, + "agent_execution_time": 42.94132041931152, + "token_usage": { + "input_tokens": 118816, + "output_tokens": 1237, + "total_tokens": 120053, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 83.3, + "avg_input_tokens": 235697.0, + "avg_output_tokens": 1723.2, + "avg_total_tokens": 237420.2, + "avg_turn_count": 12.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 256.4503297805786, + "agent_execution_time": 256.4503297805786, + "token_usage": { + "input_tokens": 974271, + "output_tokens": 2840, + "total_tokens": 977111, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 252.6647515296936, + "agent_execution_time": 252.6647515296936, + "token_usage": { + "input_tokens": 262755, + "output_tokens": 5073, + "total_tokens": 267828, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 49.02317476272583, + "agent_execution_time": 49.02317476272583, + "token_usage": { + "input_tokens": 53790, + "output_tokens": 460, + "total_tokens": 54250, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 235.77568244934082, + "agent_execution_time": 235.77568244934082, + "token_usage": { + "input_tokens": 433179, + "output_tokens": 2555, + "total_tokens": 435734, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 198.48, + "avg_input_tokens": 430998.8, + "avg_output_tokens": 2732.0, + "avg_total_tokens": 433730.8, + "avg_turn_count": 14.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 62.56175231933594, + "agent_execution_time": 62.56175231933594, + "token_usage": { + "input_tokens": 266025, + "output_tokens": 5208, + "total_tokens": 271233, + "reasoning_tokens": 3514 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 39.90877628326416, + "agent_execution_time": 39.90877628326416, + "token_usage": { + "input_tokens": 108145, + "output_tokens": 5346, + "total_tokens": 113491, + "reasoning_tokens": 3407 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 55.826473236083984, + "agent_execution_time": 55.826473236083984, + "token_usage": { + "input_tokens": 247098, + "output_tokens": 4732, + "total_tokens": 251830, + "reasoning_tokens": 3392 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 82.0383071899414, + "agent_execution_time": 82.0383071899414, + "token_usage": { + "input_tokens": 375514, + "output_tokens": 6312, + "total_tokens": 381826, + "reasoning_tokens": 3977 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 60.08, + "avg_input_tokens": 249195.5, + "avg_output_tokens": 5399.5, + "avg_total_tokens": 254595.0, + "avg_turn_count": 16.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/legal_document__dispute_review.json b/task_results/legal_document__dispute_review.json index f7e59ebd..32e4dd79 100644 --- a/task_results/legal_document__dispute_review.json +++ b/task_results/legal_document__dispute_review.json @@ -2,6 +2,73 @@ "task": "legal_document__dispute_review", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 76.4704864025116, + "agent_execution_time": 76.4704864025116, + "token_usage": { + "input_tokens": 197095, + "output_tokens": 647, + "total_tokens": 197742, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 649.4551486968994, + "agent_execution_time": 649.4551486968994, + "token_usage": { + "input_tokens": 163336, + "output_tokens": 14215, + "total_tokens": 177551, + "reasoning_tokens": 13394 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 205.48431158065796, + "agent_execution_time": 205.48431158065796, + "token_usage": { + "input_tokens": 17237, + "output_tokens": 5036, + "total_tokens": 22273, + "reasoning_tokens": 4824 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": true, + "execution_time": 518.650547504425, + "agent_execution_time": 518.650547504425, + "token_usage": { + "input_tokens": 163312, + "output_tokens": 12786, + "total_tokens": 176098, + "reasoning_tokens": 12258 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 362.52, + "avg_input_tokens": 135245.0, + "avg_output_tokens": 8171.0, + "avg_total_tokens": 143416.0, + "avg_turn_count": 6.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 124.05476307868958, - "agent_execution_time": 124.05476307868958, - "token_usage": { - "input_tokens": 144729, - "output_tokens": 1415, - "total_tokens": 146144, - "reasoning_tokens": null - }, - "turn_count": 7 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 124.05, - "avg_input_tokens": 144729.0, - "avg_output_tokens": 1415.0, - "avg_total_tokens": 146144.0, - "avg_turn_count": 7.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 45.63363075256348, + "agent_execution_time": 45.63363075256348, + "token_usage": { + "input_tokens": 154615, + "output_tokens": 1700, + "total_tokens": 156315, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 39.39472794532776, + "agent_execution_time": 39.39472794532776, + "token_usage": { + "input_tokens": 154549, + "output_tokens": 1542, + "total_tokens": 156091, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 43.42288160324097, + "agent_execution_time": 43.42288160324097, + "token_usage": { + "input_tokens": 154625, + "output_tokens": 1639, + "total_tokens": 156264, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": true, + "execution_time": 52.20779824256897, + "agent_execution_time": 52.20779824256897, + "token_usage": { + "input_tokens": 154136, + "output_tokens": 1641, + "total_tokens": 155777, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 45.16, + "avg_input_tokens": 154481.2, + "avg_output_tokens": 1630.5, + "avg_total_tokens": 156111.8, + "avg_turn_count": 7.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 54.23996448516846, + "agent_execution_time": 54.23996448516846, + "token_usage": { + "input_tokens": 193401, + "output_tokens": 1785, + "total_tokens": 195186, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 28.6076557636261, + "agent_execution_time": 28.6076557636261, + "token_usage": { + "input_tokens": 132418, + "output_tokens": 1321, + "total_tokens": 133739, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": true, + "execution_time": 54.23996448516846, + "agent_execution_time": 54.23996448516846, + "token_usage": { + "input_tokens": 193401, + "output_tokens": 1785, + "total_tokens": 195186, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 28.6076557636261, + "agent_execution_time": 28.6076557636261, + "token_usage": { + "input_tokens": 132418, + "output_tokens": 1321, + "total_tokens": 133739, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 41.42, + "avg_input_tokens": 162909.5, + "avg_output_tokens": 1553.0, + "avg_total_tokens": 164462.5, + "avg_turn_count": 7.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 78.72651314735413, + "agent_execution_time": 78.72651314735413, + "token_usage": { + "input_tokens": 197089, + "output_tokens": 650, + "total_tokens": 197739, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 86.39774417877197, + "agent_execution_time": 86.39774417877197, + "token_usage": { + "input_tokens": 134922, + "output_tokens": 982, + "total_tokens": 135904, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 114.42466068267822, + "agent_execution_time": 114.42466068267822, + "token_usage": { + "input_tokens": 139796, + "output_tokens": 614, + "total_tokens": 140410, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 126.41040921211243, + "agent_execution_time": 126.41040921211243, + "token_usage": { + "input_tokens": 140322, + "output_tokens": 1224, + "total_tokens": 141546, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 101.49, + "avg_input_tokens": 153032.2, + "avg_output_tokens": 867.5, + "avg_total_tokens": 153899.8, + "avg_turn_count": 6.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 23.48933219909668, + "agent_execution_time": 23.48933219909668, + "token_usage": { + "input_tokens": 137806, + "output_tokens": 1871, + "total_tokens": 139677, + "reasoning_tokens": 1425 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 29.011127710342407, + "agent_execution_time": 29.011127710342407, + "token_usage": { + "input_tokens": 201190, + "output_tokens": 2389, + "total_tokens": 203579, + "reasoning_tokens": 1904 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": true, + "execution_time": 24.237136840820312, + "agent_execution_time": 24.237136840820312, + "token_usage": { + "input_tokens": 136765, + "output_tokens": 2726, + "total_tokens": 139491, + "reasoning_tokens": 2336 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 37.84913921356201, + "agent_execution_time": 37.84913921356201, + "token_usage": { + "input_tokens": 151078, + "output_tokens": 3277, + "total_tokens": 154355, + "reasoning_tokens": 2772 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 28.65, + "avg_input_tokens": 156709.8, + "avg_output_tokens": 2565.8, + "avg_total_tokens": 159275.5, + "avg_turn_count": 7.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/legal_document__individual_comments.json b/task_results/legal_document__individual_comments.json index 629ba850..9ef5d0a6 100644 --- a/task_results/legal_document__individual_comments.json +++ b/task_results/legal_document__individual_comments.json @@ -2,6 +2,73 @@ "task": "legal_document__individual_comments", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 164.35221314430237, + "agent_execution_time": 164.35221314430237, + "token_usage": { + "input_tokens": 260063, + "output_tokens": 2280, + "total_tokens": 262343, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": true, + "execution_time": 676.6143836975098, + "agent_execution_time": 676.6143836975098, + "token_usage": { + "input_tokens": 179927, + "output_tokens": 15412, + "total_tokens": 195339, + "reasoning_tokens": 14694 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 3041.0798683166504, + "agent_execution_time": 3041.0798683166504, + "token_usage": { + "input_tokens": 944952, + "output_tokens": 61653, + "total_tokens": 1006605, + "reasoning_tokens": 59085 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": true, + "execution_time": 390.39178919792175, + "agent_execution_time": 390.39178919792175, + "token_usage": { + "input_tokens": 179904, + "output_tokens": 9799, + "total_tokens": 189703, + "reasoning_tokens": 9042 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 1068.11, + "avg_input_tokens": 391211.5, + "avg_output_tokens": 22286.0, + "avg_total_tokens": 413497.5, + "avg_turn_count": 8.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 156.1454041004181, - "agent_execution_time": 156.1454041004181, - "token_usage": { - "input_tokens": 193146, - "output_tokens": 1875, - "total_tokens": 195021, - "reasoning_tokens": null - }, - "turn_count": 7 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 156.15, - "avg_input_tokens": 193146.0, - "avg_output_tokens": 1875.0, - "avg_total_tokens": 195021.0, - "avg_turn_count": 7.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 77.71043944358826, + "agent_execution_time": 77.71043944358826, + "token_usage": { + "input_tokens": 195433, + "output_tokens": 2195, + "total_tokens": 197628, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": true, + "execution_time": 51.940072774887085, + "agent_execution_time": 51.940072774887085, + "token_usage": { + "input_tokens": 199357, + "output_tokens": 2166, + "total_tokens": 201523, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 50.997467041015625, + "agent_execution_time": 50.997467041015625, + "token_usage": { + "input_tokens": 195373, + "output_tokens": 2296, + "total_tokens": 197669, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 40.61216926574707, + "agent_execution_time": 40.61216926574707, + "token_usage": { + "input_tokens": 198656, + "output_tokens": 1591, + "total_tokens": 200247, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 55.32, + "avg_input_tokens": 197204.8, + "avg_output_tokens": 2062.0, + "avg_total_tokens": 199266.8, + "avg_turn_count": 6.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 87.5994017124176, + "agent_execution_time": 87.5994017124176, + "token_usage": { + "input_tokens": 254346, + "output_tokens": 3925, + "total_tokens": 258271, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 32.986831188201904, + "agent_execution_time": 32.986831188201904, + "token_usage": { + "input_tokens": 171583, + "output_tokens": 1930, + "total_tokens": 173513, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 87.5994017124176, + "agent_execution_time": 87.5994017124176, + "token_usage": { + "input_tokens": 254346, + "output_tokens": 3925, + "total_tokens": 258271, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 32.986831188201904, + "agent_execution_time": 32.986831188201904, + "token_usage": { + "input_tokens": 171583, + "output_tokens": 1930, + "total_tokens": 173513, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 60.29, + "avg_input_tokens": 212964.5, + "avg_output_tokens": 2927.5, + "avg_total_tokens": 215892.0, + "avg_turn_count": 7.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 140.80630946159363, + "agent_execution_time": 140.80630946159363, + "token_usage": { + "input_tokens": 175736, + "output_tokens": 1926, + "total_tokens": 177662, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 238.32636213302612, + "agent_execution_time": 238.32636213302612, + "token_usage": { + "input_tokens": 1172588, + "output_tokens": 2636, + "total_tokens": 1175224, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 119.4811179637909, + "agent_execution_time": 119.4811179637909, + "token_usage": { + "input_tokens": 96756, + "output_tokens": 1326, + "total_tokens": 98082, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 86.98640394210815, + "agent_execution_time": 86.98640394210815, + "token_usage": { + "input_tokens": 100961, + "output_tokens": 607, + "total_tokens": 101568, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 146.4, + "avg_input_tokens": 386510.2, + "avg_output_tokens": 1623.8, + "avg_total_tokens": 388134.0, + "avg_turn_count": 8.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 36.647098779678345, + "agent_execution_time": 36.647098779678345, + "token_usage": { + "input_tokens": 303404, + "output_tokens": 3149, + "total_tokens": 306553, + "reasoning_tokens": 2614 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 38.81229066848755, + "agent_execution_time": 38.81229066848755, + "token_usage": { + "input_tokens": 389333, + "output_tokens": 3489, + "total_tokens": 392822, + "reasoning_tokens": 2716 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 31.852614641189575, + "agent_execution_time": 31.852614641189575, + "token_usage": { + "input_tokens": 180734, + "output_tokens": 3287, + "total_tokens": 184021, + "reasoning_tokens": 2736 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 40.06143379211426, + "agent_execution_time": 40.06143379211426, + "token_usage": { + "input_tokens": 306461, + "output_tokens": 3161, + "total_tokens": 309622, + "reasoning_tokens": 2599 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 36.84, + "avg_input_tokens": 294983.0, + "avg_output_tokens": 3271.5, + "avg_total_tokens": 298254.5, + "avg_turn_count": 9.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/legal_document__solution_tracing.json b/task_results/legal_document__solution_tracing.json index 0ee69fd2..75da4f88 100644 --- a/task_results/legal_document__solution_tracing.json +++ b/task_results/legal_document__solution_tracing.json @@ -2,6 +2,73 @@ "task": "legal_document__solution_tracing", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 130.7681760787964, + "agent_execution_time": 130.7681760787964, + "token_usage": { + "input_tokens": 215375, + "output_tokens": 1409, + "total_tokens": 216784, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 440.03814792633057, + "agent_execution_time": 440.03814792633057, + "token_usage": { + "input_tokens": 18018, + "output_tokens": 3404, + "total_tokens": 21422, + "reasoning_tokens": 2864 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 100.01496195793152, + "agent_execution_time": 100.01496195793152, + "token_usage": { + "input_tokens": 6747, + "output_tokens": 2365, + "total_tokens": 9112, + "reasoning_tokens": 2330 + }, + "turn_count": 2 + }, + { + "run": "run-4", + "success": false, + "execution_time": 237.7526581287384, + "agent_execution_time": 237.7526581287384, + "token_usage": { + "input_tokens": 14140, + "output_tokens": 4602, + "total_tokens": 18742, + "reasoning_tokens": 4110 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 227.14, + "avg_input_tokens": 63570.0, + "avg_output_tokens": 2945.0, + "avg_total_tokens": 66515.0, + "avg_turn_count": 4.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 143.57952189445496, - "agent_execution_time": 143.57952189445496, - "token_usage": { - "input_tokens": 294946, - "output_tokens": 1614, - "total_tokens": 296560, - "reasoning_tokens": null - }, - "turn_count": 8 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 143.58, - "avg_input_tokens": 294946.0, - "avg_output_tokens": 1614.0, - "avg_total_tokens": 296560.0, - "avg_turn_count": 8.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 55.318326234817505, + "agent_execution_time": 55.318326234817505, + "token_usage": { + "input_tokens": 286493, + "output_tokens": 1998, + "total_tokens": 288491, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 50.05606198310852, + "agent_execution_time": 50.05606198310852, + "token_usage": { + "input_tokens": 286573, + "output_tokens": 1895, + "total_tokens": 288468, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 49.31079387664795, + "agent_execution_time": 49.31079387664795, + "token_usage": { + "input_tokens": 286475, + "output_tokens": 1794, + "total_tokens": 288269, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 50.229806423187256, + "agent_execution_time": 50.229806423187256, + "token_usage": { + "input_tokens": 286461, + "output_tokens": 1827, + "total_tokens": 288288, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 51.23, + "avg_input_tokens": 286500.5, + "avg_output_tokens": 1878.5, + "avg_total_tokens": 288379.0, + "avg_turn_count": 7.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 42.35612201690674, + "agent_execution_time": 42.35612201690674, + "token_usage": { + "input_tokens": 209836, + "output_tokens": 1634, + "total_tokens": 211470, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 42.90608882904053, + "agent_execution_time": 42.90608882904053, + "token_usage": { + "input_tokens": 210014, + "output_tokens": 1579, + "total_tokens": 211593, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 42.35612201690674, + "agent_execution_time": 42.35612201690674, + "token_usage": { + "input_tokens": 209836, + "output_tokens": 1634, + "total_tokens": 211470, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 42.90608882904053, + "agent_execution_time": 42.90608882904053, + "token_usage": { + "input_tokens": 210014, + "output_tokens": 1579, + "total_tokens": 211593, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 42.63, + "avg_input_tokens": 209925.0, + "avg_output_tokens": 1606.5, + "avg_total_tokens": 211531.5, + "avg_turn_count": 7.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 132.0972511768341, + "agent_execution_time": 132.0972511768341, + "token_usage": { + "input_tokens": 215309, + "output_tokens": 1297, + "total_tokens": 216606, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 172.6146595478058, + "agent_execution_time": 172.6146595478058, + "token_usage": { + "input_tokens": 356709, + "output_tokens": 1544, + "total_tokens": 358253, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 267.18336248397827, + "agent_execution_time": 267.18336248397827, + "token_usage": { + "input_tokens": 322942, + "output_tokens": 1151, + "total_tokens": 324093, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 373.6008851528168, + "agent_execution_time": 373.6008851528168, + "token_usage": { + "input_tokens": 419197, + "output_tokens": 2241, + "total_tokens": 421438, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 236.37, + "avg_input_tokens": 328539.2, + "avg_output_tokens": 1558.2, + "avg_total_tokens": 330097.5, + "avg_turn_count": 6.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 49.13685917854309, + "agent_execution_time": 49.13685917854309, + "token_usage": { + "input_tokens": 379391, + "output_tokens": 3958, + "total_tokens": 383349, + "reasoning_tokens": 3339 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": true, + "execution_time": 49.26004719734192, + "agent_execution_time": 49.26004719734192, + "token_usage": { + "input_tokens": 378323, + "output_tokens": 4112, + "total_tokens": 382435, + "reasoning_tokens": 3491 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": true, + "execution_time": 42.50874185562134, + "agent_execution_time": 42.50874185562134, + "token_usage": { + "input_tokens": 402227, + "output_tokens": 4112, + "total_tokens": 406339, + "reasoning_tokens": 3471 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 43.66520857810974, + "agent_execution_time": 43.66520857810974, + "token_usage": { + "input_tokens": 404606, + "output_tokens": 4548, + "total_tokens": 409154, + "reasoning_tokens": 3811 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 46.14, + "avg_input_tokens": 391136.8, + "avg_output_tokens": 4182.5, + "avg_total_tokens": 395319.2, + "avg_turn_count": 8.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/lego__consistency_enforcement.json b/task_results/lego__consistency_enforcement.json index bd42279f..3c8ba031 100644 --- a/task_results/lego__consistency_enforcement.json +++ b/task_results/lego__consistency_enforcement.json @@ -2,6 +2,73 @@ "task": "lego__consistency_enforcement", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 259.01617765426636, + "agent_execution_time": 259.01617765426636, + "token_usage": { + "input_tokens": 722407, + "output_tokens": 3651, + "total_tokens": 726058, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": true, + "execution_time": 319.63316440582275, + "agent_execution_time": 319.63316440582275, + "token_usage": { + "input_tokens": 1151155, + "output_tokens": 4392, + "total_tokens": 1155547, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": true, + "execution_time": 1277.2306110858917, + "agent_execution_time": 1277.2306110858917, + "token_usage": { + "input_tokens": 324866, + "output_tokens": 27303, + "total_tokens": 352169, + "reasoning_tokens": 23920 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": true, + "execution_time": 532.2614402770996, + "agent_execution_time": 532.2614402770996, + "token_usage": { + "input_tokens": 68391, + "output_tokens": 13364, + "total_tokens": 81755, + "reasoning_tokens": 12151 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 597.04, + "avg_input_tokens": 566704.8, + "avg_output_tokens": 12177.5, + "avg_total_tokens": 578882.2, + "avg_turn_count": 13.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 393.2954270839691, - "agent_execution_time": 393.2954270839691, - "token_usage": { - "input_tokens": 172724, - "output_tokens": 7067, - "total_tokens": 179791, - "reasoning_tokens": null - }, - "turn_count": 25 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 393.3, - "avg_input_tokens": 172724.0, - "avg_output_tokens": 7067.0, - "avg_total_tokens": 179791.0, - "avg_turn_count": 25.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 203.31866574287415, + "agent_execution_time": 203.31866574287415, + "token_usage": { + "input_tokens": 336058, + "output_tokens": 7078, + "total_tokens": 343136, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": true, + "execution_time": 212.3275921344757, + "agent_execution_time": 212.3275921344757, + "token_usage": { + "input_tokens": 1649469, + "output_tokens": 7648, + "total_tokens": 1657117, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-3", + "success": true, + "execution_time": 176.8132345676422, + "agent_execution_time": 176.8132345676422, + "token_usage": { + "input_tokens": 518130, + "output_tokens": 7826, + "total_tokens": 525956, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-4", + "success": true, + "execution_time": 214.29601955413818, + "agent_execution_time": 214.29601955413818, + "token_usage": { + "input_tokens": 1491130, + "output_tokens": 8038, + "total_tokens": 1499168, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 201.69, + "avg_input_tokens": 998696.8, + "avg_output_tokens": 7647.5, + "avg_total_tokens": 1006344.2, + "avg_turn_count": 27.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 338.54951190948486, + "agent_execution_time": 338.54951190948486, + "token_usage": { + "input_tokens": 1087510, + "output_tokens": 6107, + "total_tokens": 1093617, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": true, + "execution_time": 917.103600025177, + "agent_execution_time": 917.103600025177, + "token_usage": { + "input_tokens": 1636709, + "output_tokens": 7481, + "total_tokens": 1644190, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-3", + "success": true, + "execution_time": 519.8713345527649, + "agent_execution_time": 519.8713345527649, + "token_usage": { + "input_tokens": 1684524, + "output_tokens": 10347, + "total_tokens": 1694871, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-4", + "success": true, + "execution_time": 191.10808682441711, + "agent_execution_time": 191.10808682441711, + "token_usage": { + "input_tokens": 332968, + "output_tokens": 6512, + "total_tokens": 339480, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 491.66, + "avg_input_tokens": 1185427.8, + "avg_output_tokens": 7611.8, + "avg_total_tokens": 1193039.5, + "avg_turn_count": 25.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 332.5333845615387, + "agent_execution_time": 332.5333845615387, + "token_usage": { + "input_tokens": 1315257, + "output_tokens": 4759, + "total_tokens": 1320016, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-2", + "success": false, + "execution_time": 189.04312682151794, + "agent_execution_time": 189.04312682151794, + "token_usage": { + "input_tokens": 444134, + "output_tokens": 1721, + "total_tokens": 445855, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 640.5812239646912, + "agent_execution_time": 640.5812239646912, + "token_usage": { + "input_tokens": 1399627, + "output_tokens": 5161, + "total_tokens": 1404788, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.20232081413269, + "agent_execution_time": 8.20232081413269, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 292.59, + "avg_input_tokens": 789754.5, + "avg_output_tokens": 2910.2, + "avg_total_tokens": 792664.8, + "avg_turn_count": 16.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 68.30837321281433, + "agent_execution_time": 68.30837321281433, + "token_usage": { + "input_tokens": 481345, + "output_tokens": 5469, + "total_tokens": 486814, + "reasoning_tokens": 3546 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 104.10069179534912, + "agent_execution_time": 104.10069179534912, + "token_usage": { + "input_tokens": 451890, + "output_tokens": 8675, + "total_tokens": 460565, + "reasoning_tokens": 6453 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": true, + "execution_time": 79.97889614105225, + "agent_execution_time": 79.97889614105225, + "token_usage": { + "input_tokens": 276928, + "output_tokens": 6839, + "total_tokens": 283767, + "reasoning_tokens": 3607 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": true, + "execution_time": 69.09242391586304, + "agent_execution_time": 69.09242391586304, + "token_usage": { + "input_tokens": 430884, + "output_tokens": 6994, + "total_tokens": 437878, + "reasoning_tokens": 3735 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 80.37, + "avg_input_tokens": 410261.8, + "avg_output_tokens": 6994.2, + "avg_total_tokens": 417256.0, + "avg_turn_count": 14.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/lego__database_security_policies.json b/task_results/lego__database_security_policies.json index c6a6816d..82d9212f 100644 --- a/task_results/lego__database_security_policies.json +++ b/task_results/lego__database_security_policies.json @@ -2,6 +2,73 @@ "task": "lego__database_security_policies", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 300.08112144470215, + "agent_execution_time": 300.08112144470215, + "token_usage": { + "input_tokens": 177379, + "output_tokens": 4401, + "total_tokens": 181780, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": true, + "execution_time": 539.3631503582001, + "agent_execution_time": 539.3631503582001, + "token_usage": { + "input_tokens": 381895, + "output_tokens": 8235, + "total_tokens": 390130, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + { + "run": "run-3", + "success": true, + "execution_time": 929.1428561210632, + "agent_execution_time": 929.1428561210632, + "token_usage": { + "input_tokens": 100425, + "output_tokens": 24177, + "total_tokens": 124602, + "reasoning_tokens": 21045 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 331.6807019710541, + "agent_execution_time": 331.6807019710541, + "token_usage": { + "input_tokens": 10009, + "output_tokens": 7753, + "total_tokens": 17762, + "reasoning_tokens": 7677 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 525.07, + "avg_input_tokens": 167427.0, + "avg_output_tokens": 11141.5, + "avg_total_tokens": 178568.5, + "avg_turn_count": 22.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 888.4085941314697, - "agent_execution_time": 888.4085941314697, - "token_usage": { - "input_tokens": 353413, - "output_tokens": 18779, - "total_tokens": 372192, - "reasoning_tokens": null - }, - "turn_count": 30 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 888.41, - "avg_input_tokens": 353413.0, - "avg_output_tokens": 18779.0, - "avg_total_tokens": 372192.0, - "avg_turn_count": 30.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 403.9742548465729, + "agent_execution_time": 403.9742548465729, + "token_usage": { + "input_tokens": 714649, + "output_tokens": 13742, + "total_tokens": 728391, + "reasoning_tokens": 0 + }, + "turn_count": 43 + }, + { + "run": "run-2", + "success": true, + "execution_time": 108.09306788444519, + "agent_execution_time": 108.09306788444519, + "token_usage": { + "input_tokens": 196827, + "output_tokens": 3976, + "total_tokens": 200803, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": true, + "execution_time": 111.85084342956543, + "agent_execution_time": 111.85084342956543, + "token_usage": { + "input_tokens": 231728, + "output_tokens": 3763, + "total_tokens": 235491, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 220.7334008216858, + "agent_execution_time": 220.7334008216858, + "token_usage": { + "input_tokens": 370627, + "output_tokens": 11738, + "total_tokens": 382365, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 211.16, + "avg_input_tokens": 378457.8, + "avg_output_tokens": 8304.8, + "avg_total_tokens": 386762.5, + "avg_turn_count": 28.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 109.89619183540344, + "agent_execution_time": 109.89619183540344, + "token_usage": { + "input_tokens": 137413, + "output_tokens": 3005, + "total_tokens": 140418, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 489.69895696640015, + "agent_execution_time": 489.69895696640015, + "token_usage": { + "input_tokens": 334309, + "output_tokens": 7466, + "total_tokens": 341775, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-3", + "success": false, + "execution_time": 157.25187420845032, + "agent_execution_time": 157.25187420845032, + "token_usage": { + "input_tokens": 241924, + "output_tokens": 3962, + "total_tokens": 245886, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-4", + "success": true, + "execution_time": 104.91177606582642, + "agent_execution_time": 104.91177606582642, + "token_usage": { + "input_tokens": 139041, + "output_tokens": 3023, + "total_tokens": 142064, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 215.44, + "avg_input_tokens": 213171.8, + "avg_output_tokens": 4364.0, + "avg_total_tokens": 217535.8, + "avg_turn_count": 26.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 195.53896188735962, + "agent_execution_time": 195.53896188735962, + "token_usage": { + "input_tokens": 172815, + "output_tokens": 2186, + "total_tokens": 175001, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 802.7101278305054, + "agent_execution_time": 802.7101278305054, + "token_usage": { + "input_tokens": 2309536, + "output_tokens": 10350, + "total_tokens": 2319886, + "reasoning_tokens": 0 + }, + "turn_count": 60 + }, + { + "run": "run-3", + "success": false, + "execution_time": 301.2383608818054, + "agent_execution_time": 301.2383608818054, + "token_usage": { + "input_tokens": 228041, + "output_tokens": 4329, + "total_tokens": 232370, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.234580516815186, + "agent_execution_time": 8.234580516815186, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 326.93, + "avg_input_tokens": 677598.0, + "avg_output_tokens": 4216.2, + "avg_total_tokens": 681814.2, + "avg_turn_count": 29.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 52.30773138999939, + "agent_execution_time": 52.30773138999939, + "token_usage": { + "input_tokens": 165312, + "output_tokens": 5291, + "total_tokens": 170603, + "reasoning_tokens": 1810 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": true, + "execution_time": 83.63074851036072, + "agent_execution_time": 83.63074851036072, + "token_usage": { + "input_tokens": 191979, + "output_tokens": 6961, + "total_tokens": 198940, + "reasoning_tokens": 2940 + }, + "turn_count": 26 + }, + { + "run": "run-3", + "success": true, + "execution_time": 74.1859028339386, + "agent_execution_time": 74.1859028339386, + "token_usage": { + "input_tokens": 205754, + "output_tokens": 5882, + "total_tokens": 211636, + "reasoning_tokens": 2024 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": true, + "execution_time": 74.46791768074036, + "agent_execution_time": 74.46791768074036, + "token_usage": { + "input_tokens": 258020, + "output_tokens": 6717, + "total_tokens": 264737, + "reasoning_tokens": 2612 + }, + "turn_count": 32 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 71.15, + "avg_input_tokens": 205266.2, + "avg_output_tokens": 6212.8, + "avg_total_tokens": 211479.0, + "avg_turn_count": 27.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/lego__transactional_inventory_transfer.json b/task_results/lego__transactional_inventory_transfer.json index 315d22d4..e10fd5fb 100644 --- a/task_results/lego__transactional_inventory_transfer.json +++ b/task_results/lego__transactional_inventory_transfer.json @@ -2,6 +2,73 @@ "task": "lego__transactional_inventory_transfer", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 867.2886288166046, + "agent_execution_time": 867.2886288166046, + "token_usage": { + "input_tokens": 2997259, + "output_tokens": 13824, + "total_tokens": 3011083, + "reasoning_tokens": 0 + }, + "turn_count": 50 + }, + { + "run": "run-2", + "success": true, + "execution_time": 711.7165093421936, + "agent_execution_time": 711.7165093421936, + "token_usage": { + "input_tokens": 688995, + "output_tokens": 12298, + "total_tokens": 701293, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + { + "run": "run-3", + "success": false, + "execution_time": 122.4326663017273, + "agent_execution_time": 122.4326663017273, + "token_usage": { + "input_tokens": 3125, + "output_tokens": 3010, + "total_tokens": 6135, + "reasoning_tokens": 603 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1007.8619937896729, + "agent_execution_time": 1007.8619937896729, + "token_usage": { + "input_tokens": 22228, + "output_tokens": 9828, + "total_tokens": 32056, + "reasoning_tokens": 7576 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 677.32, + "avg_input_tokens": 927901.8, + "avg_output_tokens": 9740.0, + "avg_total_tokens": 937641.8, + "avg_turn_count": 24.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 601.5841164588928, - "agent_execution_time": 601.5841164588928, - "token_usage": { - "input_tokens": 304864, - "output_tokens": 11731, - "total_tokens": 316595, - "reasoning_tokens": null - }, - "turn_count": 29 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 601.58, - "avg_input_tokens": 304864.0, - "avg_output_tokens": 11731.0, - "avg_total_tokens": 316595.0, - "avg_turn_count": 29.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 616.4718172550201, + "agent_execution_time": 616.4718172550201, + "token_usage": { + "input_tokens": 1179003, + "output_tokens": 18991, + "total_tokens": 1197994, + "reasoning_tokens": 0 + }, + "turn_count": 51 + }, + { + "run": "run-2", + "success": false, + "execution_time": 376.94758009910583, + "agent_execution_time": 376.94758009910583, + "token_usage": { + "input_tokens": 1150560, + "output_tokens": 17002, + "total_tokens": 1167562, + "reasoning_tokens": 0 + }, + "turn_count": 54 + }, + { + "run": "run-3", + "success": false, + "execution_time": 283.57749247550964, + "agent_execution_time": 283.57749247550964, + "token_usage": { + "input_tokens": 841284, + "output_tokens": 12400, + "total_tokens": 853684, + "reasoning_tokens": 0 + }, + "turn_count": 41 + }, + { + "run": "run-4", + "success": false, + "execution_time": 551.8978893756866, + "agent_execution_time": 551.8978893756866, + "token_usage": { + "input_tokens": 1863746, + "output_tokens": 25469, + "total_tokens": 1889215, + "reasoning_tokens": 0 + }, + "turn_count": 64 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 457.22, + "avg_input_tokens": 1258648.2, + "avg_output_tokens": 18465.5, + "avg_total_tokens": 1277113.8, + "avg_turn_count": 52.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 455.07564425468445, + "agent_execution_time": 455.07564425468445, + "token_usage": { + "input_tokens": 765863, + "output_tokens": 13228, + "total_tokens": 779091, + "reasoning_tokens": 0 + }, + "turn_count": 49 + }, + { + "run": "run-2", + "success": true, + "execution_time": 127.83743572235107, + "agent_execution_time": 127.83743572235107, + "token_usage": { + "input_tokens": 55998, + "output_tokens": 3507, + "total_tokens": 59505, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 570.248443365097, + "agent_execution_time": 570.248443365097, + "token_usage": { + "input_tokens": 1274653, + "output_tokens": 13148, + "total_tokens": 1287801, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-4", + "success": true, + "execution_time": 437.2577073574066, + "agent_execution_time": 437.2577073574066, + "token_usage": { + "input_tokens": 746066, + "output_tokens": 17352, + "total_tokens": 763418, + "reasoning_tokens": 0 + }, + "turn_count": 46 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 397.6, + "avg_input_tokens": 710645.0, + "avg_output_tokens": 11808.8, + "avg_total_tokens": 722453.8, + "avg_turn_count": 34.25, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 736.6222898960114, + "agent_execution_time": 736.6222898960114, + "token_usage": { + "input_tokens": 683489, + "output_tokens": 15041, + "total_tokens": 698530, + "reasoning_tokens": 0 + }, + "turn_count": 43 + }, + { + "run": "run-2", + "success": true, + "execution_time": 673.1510260105133, + "agent_execution_time": 673.1510260105133, + "token_usage": { + "input_tokens": 671371, + "output_tokens": 11592, + "total_tokens": 682963, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + { + "run": "run-3", + "success": false, + "execution_time": 908.8019723892212, + "agent_execution_time": 908.8019723892212, + "token_usage": { + "input_tokens": 927507, + "output_tokens": 16115, + "total_tokens": 943622, + "reasoning_tokens": 0 + }, + "turn_count": 49 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.176013946533203, + "agent_execution_time": 8.176013946533203, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 581.69, + "avg_input_tokens": 570591.8, + "avg_output_tokens": 10687.0, + "avg_total_tokens": 581278.8, + "avg_turn_count": 33.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 117.18111109733582, + "agent_execution_time": 117.18111109733582, + "token_usage": { + "input_tokens": 115872, + "output_tokens": 15654, + "total_tokens": 131526, + "reasoning_tokens": 13751 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 164.9237825870514, + "agent_execution_time": 164.9237825870514, + "token_usage": { + "input_tokens": 505122, + "output_tokens": 13027, + "total_tokens": 518149, + "reasoning_tokens": 9360 + }, + "turn_count": 34 + }, + { + "run": "run-3", + "success": false, + "execution_time": 96.16848468780518, + "agent_execution_time": 96.16848468780518, + "token_usage": { + "input_tokens": 36427, + "output_tokens": 8141, + "total_tokens": 44568, + "reasoning_tokens": 6385 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": true, + "execution_time": 63.01567244529724, + "agent_execution_time": 63.01567244529724, + "token_usage": { + "input_tokens": 64635, + "output_tokens": 8024, + "total_tokens": 72659, + "reasoning_tokens": 4248 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 110.32, + "avg_input_tokens": 180514.0, + "avg_output_tokens": 11211.5, + "avg_total_tokens": 191725.5, + "avg_turn_count": 15.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/mcpmark-cicd__deployment_status_workflow.json b/task_results/mcpmark-cicd__deployment_status_workflow.json index f1fe57b9..f3d4c47a 100644 --- a/task_results/mcpmark-cicd__deployment_status_workflow.json +++ b/task_results/mcpmark-cicd__deployment_status_workflow.json @@ -2,6 +2,73 @@ "task": "mcpmark-cicd__deployment_status_workflow", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 266.72085332870483, + "agent_execution_time": 266.72085332870483, + "token_usage": { + "input_tokens": 145178, + "output_tokens": 5114, + "total_tokens": 150292, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 685.8437578678131, + "agent_execution_time": 685.8437578678131, + "token_usage": { + "input_tokens": 107018, + "output_tokens": 15088, + "total_tokens": 122106, + "reasoning_tokens": 12341 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 675.1347379684448, + "agent_execution_time": 675.1347379684448, + "token_usage": { + "input_tokens": 162826, + "output_tokens": 15080, + "total_tokens": 177906, + "reasoning_tokens": 11477 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 493.03128576278687, + "agent_execution_time": 493.03128576278687, + "token_usage": { + "input_tokens": 107142, + "output_tokens": 14013, + "total_tokens": 121155, + "reasoning_tokens": 11437 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 530.18, + "avg_input_tokens": 130541.0, + "avg_output_tokens": 12323.8, + "avg_total_tokens": 142864.8, + "avg_turn_count": 5.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 2233.8593094348907, - "agent_execution_time": 2233.8593094348907, - "token_usage": { - "input_tokens": 403936, - "output_tokens": 51557, - "total_tokens": 455493, - "reasoning_tokens": 0 - }, - "turn_count": 20 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 2233.86, - "avg_input_tokens": 403936.0, - "avg_output_tokens": 51557.0, - "avg_total_tokens": 455493.0, - "avg_turn_count": 20.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 523.5952243804932, + "agent_execution_time": 523.5952243804932, + "token_usage": { + "input_tokens": 876833, + "output_tokens": 32914, + "total_tokens": 909747, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 319.86090779304504, + "agent_execution_time": 319.86090779304504, + "token_usage": { + "input_tokens": 882718, + "output_tokens": 16761, + "total_tokens": 899479, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": false, + "execution_time": 892.861257314682, + "agent_execution_time": 892.861257314682, + "token_usage": { + "input_tokens": 2449736, + "output_tokens": 49222, + "total_tokens": 2498958, + "reasoning_tokens": 0 + }, + "turn_count": 82 + }, + { + "run": "run-4", + "success": false, + "execution_time": 827.4493978023529, + "agent_execution_time": 827.4493978023529, + "token_usage": { + "input_tokens": 3133840, + "output_tokens": 43170, + "total_tokens": 3177010, + "reasoning_tokens": 0 + }, + "turn_count": 100 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 640.94, + "avg_input_tokens": 1835781.8, + "avg_output_tokens": 35516.8, + "avg_total_tokens": 1871298.5, + "avg_turn_count": 57.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 240.17909598350525, + "agent_execution_time": 240.17909598350525, + "token_usage": { + "input_tokens": 234759, + "output_tokens": 6379, + "total_tokens": 241138, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 276.39716053009033, + "agent_execution_time": 276.39716053009033, + "token_usage": { + "input_tokens": 482961, + "output_tokens": 6822, + "total_tokens": 489783, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": false, + "execution_time": 193.3067033290863, + "agent_execution_time": 193.3067033290863, + "token_usage": { + "input_tokens": 272083, + "output_tokens": 5770, + "total_tokens": 277853, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 188.82461166381836, + "agent_execution_time": 188.82461166381836, + "token_usage": { + "input_tokens": 280023, + "output_tokens": 7106, + "total_tokens": 287129, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 224.68, + "avg_input_tokens": 317456.5, + "avg_output_tokens": 6519.2, + "avg_total_tokens": 323975.8, + "avg_turn_count": 18.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 233.63362741470337, + "agent_execution_time": 233.63362741470337, + "token_usage": { + "input_tokens": 101102, + "output_tokens": 4806, + "total_tokens": 105908, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.76741623878479, + "agent_execution_time": 10.76741623878479, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 249.43173933029175, + "agent_execution_time": 249.43173933029175, + "token_usage": { + "input_tokens": 136980, + "output_tokens": 4722, + "total_tokens": 141702, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 113.7837073802948, + "agent_execution_time": 113.7837073802948, + "token_usage": { + "input_tokens": 48735, + "output_tokens": 3097, + "total_tokens": 51832, + "reasoning_tokens": 0 + }, + "turn_count": 2 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 151.9, + "avg_input_tokens": 71704.2, + "avg_output_tokens": 3156.2, + "avg_total_tokens": 74860.5, + "avg_turn_count": 3.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 57.17888879776001, + "agent_execution_time": 57.17888879776001, + "token_usage": { + "input_tokens": 86080, + "output_tokens": 4983, + "total_tokens": 91063, + "reasoning_tokens": 2856 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 37.55769395828247, + "agent_execution_time": 37.55769395828247, + "token_usage": { + "input_tokens": 84602, + "output_tokens": 4491, + "total_tokens": 89093, + "reasoning_tokens": 2609 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 58.69420385360718, + "agent_execution_time": 58.69420385360718, + "token_usage": { + "input_tokens": 93903, + "output_tokens": 6910, + "total_tokens": 100813, + "reasoning_tokens": 4900 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 69.0181930065155, + "agent_execution_time": 69.0181930065155, + "token_usage": { + "input_tokens": 87421, + "output_tokens": 5443, + "total_tokens": 92864, + "reasoning_tokens": 3386 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 55.61, + "avg_input_tokens": 88001.5, + "avg_output_tokens": 5456.8, + "avg_total_tokens": 93458.2, + "avg_turn_count": 5.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/mcpmark-cicd__issue_management_workflow.json b/task_results/mcpmark-cicd__issue_management_workflow.json index 7ff89665..cac2ccd7 100644 --- a/task_results/mcpmark-cicd__issue_management_workflow.json +++ b/task_results/mcpmark-cicd__issue_management_workflow.json @@ -2,6 +2,73 @@ "task": "mcpmark-cicd__issue_management_workflow", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 537.5137591362, + "agent_execution_time": 537.5137591362, + "token_usage": { + "input_tokens": 1028661, + "output_tokens": 7924, + "total_tokens": 1036585, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1096.5259656906128, + "agent_execution_time": 1096.5259656906128, + "token_usage": { + "input_tokens": 219938, + "output_tokens": 23213, + "total_tokens": 243151, + "reasoning_tokens": 17906 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1718.2038402557373, + "agent_execution_time": 1718.2038402557373, + "token_usage": { + "input_tokens": 338588, + "output_tokens": 21236, + "total_tokens": 359824, + "reasoning_tokens": 16802 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 791.8517625331879, + "agent_execution_time": 791.8517625331879, + "token_usage": { + "input_tokens": 110326, + "output_tokens": 22859, + "total_tokens": 133185, + "reasoning_tokens": 18835 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1036.02, + "avg_input_tokens": 424378.2, + "avg_output_tokens": 18808.0, + "avg_total_tokens": 443186.2, + "avg_turn_count": 12.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 531.0920813083649, - "agent_execution_time": 531.0920813083649, - "token_usage": { - "input_tokens": 551218, - "output_tokens": 8782, - "total_tokens": 560000, - "reasoning_tokens": 0 - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 531.09, - "avg_input_tokens": 551218.0, - "avg_output_tokens": 8782.0, - "avg_total_tokens": 560000.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 353.6409010887146, + "agent_execution_time": 353.6409010887146, + "token_usage": { + "input_tokens": 648047, + "output_tokens": 21611, + "total_tokens": 669658, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 567.3189604282379, + "agent_execution_time": 567.3189604282379, + "token_usage": { + "input_tokens": 1427391, + "output_tokens": 17732, + "total_tokens": 1445123, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-3", + "success": false, + "execution_time": 504.7355761528015, + "agent_execution_time": 504.7355761528015, + "token_usage": { + "input_tokens": 771229, + "output_tokens": 33138, + "total_tokens": 804367, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 556.5370657444, + "agent_execution_time": 556.5370657444, + "token_usage": { + "input_tokens": 1058191, + "output_tokens": 34557, + "total_tokens": 1092748, + "reasoning_tokens": 0 + }, + "turn_count": 28 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 495.56, + "avg_input_tokens": 976214.5, + "avg_output_tokens": 26759.5, + "avg_total_tokens": 1002974.0, + "avg_turn_count": 24.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 840.6754078865051, + "agent_execution_time": 840.6754078865051, + "token_usage": { + "input_tokens": 2258512, + "output_tokens": 16513, + "total_tokens": 2275025, + "reasoning_tokens": 0 + }, + "turn_count": 61 + }, + { + "run": "run-2", + "success": false, + "execution_time": 359.1804759502411, + "agent_execution_time": 359.1804759502411, + "token_usage": { + "input_tokens": 1088007, + "output_tokens": 9824, + "total_tokens": 1097831, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + { + "run": "run-3", + "success": false, + "execution_time": 506.37666034698486, + "agent_execution_time": 506.37666034698486, + "token_usage": { + "input_tokens": 2684040, + "output_tokens": 12887, + "total_tokens": 2696927, + "reasoning_tokens": 0 + }, + "turn_count": 52 + }, + { + "run": "run-4", + "success": false, + "execution_time": 589.6776521205902, + "agent_execution_time": 589.6776521205902, + "token_usage": { + "input_tokens": 3772829, + "output_tokens": 8350, + "total_tokens": 3781179, + "reasoning_tokens": 0 + }, + "turn_count": 62 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 573.98, + "avg_input_tokens": 2450847.0, + "avg_output_tokens": 11893.5, + "avg_total_tokens": 2462740.5, + "avg_turn_count": 53.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 525.1548409461975, + "agent_execution_time": 525.1548409461975, + "token_usage": { + "input_tokens": 1529635, + "output_tokens": 7300, + "total_tokens": 1536935, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-2", + "success": false, + "execution_time": 11.621798515319824, + "agent_execution_time": 11.621798515319824, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 307.3473916053772, + "agent_execution_time": 307.3473916053772, + "token_usage": { + "input_tokens": 226627, + "output_tokens": 4940, + "total_tokens": 231567, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 464.826598405838, + "agent_execution_time": 464.826598405838, + "token_usage": { + "input_tokens": 505330, + "output_tokens": 8633, + "total_tokens": 513963, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 327.24, + "avg_input_tokens": 565398.0, + "avg_output_tokens": 5218.2, + "avg_total_tokens": 570616.2, + "avg_turn_count": 12.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 81.23488736152649, + "agent_execution_time": 81.23488736152649, + "token_usage": { + "input_tokens": 232946, + "output_tokens": 8666, + "total_tokens": 241612, + "reasoning_tokens": 4841 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 99.36858129501343, + "agent_execution_time": 99.36858129501343, + "token_usage": { + "input_tokens": 261304, + "output_tokens": 9938, + "total_tokens": 271242, + "reasoning_tokens": 6376 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 82.35587882995605, + "agent_execution_time": 82.35587882995605, + "token_usage": { + "input_tokens": 166010, + "output_tokens": 11508, + "total_tokens": 177518, + "reasoning_tokens": 7859 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 82.02052307128906, + "agent_execution_time": 82.02052307128906, + "token_usage": { + "input_tokens": 254939, + "output_tokens": 10525, + "total_tokens": 265464, + "reasoning_tokens": 6807 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 86.24, + "avg_input_tokens": 228799.8, + "avg_output_tokens": 10159.2, + "avg_total_tokens": 238959.0, + "avg_turn_count": 10.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/mcpmark-cicd__linting_ci_workflow.json b/task_results/mcpmark-cicd__linting_ci_workflow.json index ae6ada74..fc2faa38 100644 --- a/task_results/mcpmark-cicd__linting_ci_workflow.json +++ b/task_results/mcpmark-cicd__linting_ci_workflow.json @@ -2,6 +2,73 @@ "task": "mcpmark-cicd__linting_ci_workflow", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 119.7724871635437, + "agent_execution_time": 119.7724871635437, + "token_usage": { + "input_tokens": 133512, + "output_tokens": 1688, + "total_tokens": 135200, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 379.9204249382019, + "agent_execution_time": 379.9204249382019, + "token_usage": { + "input_tokens": 49738, + "output_tokens": 8380, + "total_tokens": 58118, + "reasoning_tokens": 7401 + }, + "turn_count": 2 + }, + { + "run": "run-3", + "success": false, + "execution_time": 786.3401265144348, + "agent_execution_time": 786.3401265144348, + "token_usage": { + "input_tokens": 182005, + "output_tokens": 15134, + "total_tokens": 197139, + "reasoning_tokens": 13585 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": true, + "execution_time": 462.70820474624634, + "agent_execution_time": 462.70820474624634, + "token_usage": { + "input_tokens": 101168, + "output_tokens": 13098, + "total_tokens": 114266, + "reasoning_tokens": 11758 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 437.19, + "avg_input_tokens": 116605.8, + "avg_output_tokens": 9575.0, + "avg_total_tokens": 126180.8, + "avg_turn_count": 5.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 174.83519554138184, - "agent_execution_time": 174.83519554138184, - "token_usage": { - "input_tokens": 155317, - "output_tokens": 2781, - "total_tokens": 158098, - "reasoning_tokens": null - }, - "turn_count": 7 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 174.84, - "avg_input_tokens": 155317.0, - "avg_output_tokens": 2781.0, - "avg_total_tokens": 158098.0, - "avg_turn_count": 7.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 80.46243286132812, + "agent_execution_time": 80.46243286132812, + "token_usage": { + "input_tokens": 196962, + "output_tokens": 3144, + "total_tokens": 200106, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 73.94100975990295, + "agent_execution_time": 73.94100975990295, + "token_usage": { + "input_tokens": 174836, + "output_tokens": 2907, + "total_tokens": 177743, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": true, + "execution_time": 139.19495487213135, + "agent_execution_time": 139.19495487213135, + "token_usage": { + "input_tokens": 209881, + "output_tokens": 2957, + "total_tokens": 212838, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": true, + "execution_time": 76.99708795547485, + "agent_execution_time": 76.99708795547485, + "token_usage": { + "input_tokens": 184479, + "output_tokens": 2794, + "total_tokens": 187273, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 92.65, + "avg_input_tokens": 191539.5, + "avg_output_tokens": 2950.5, + "avg_total_tokens": 194490.0, + "avg_turn_count": 8.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 111.29600739479065, + "agent_execution_time": 111.29600739479065, + "token_usage": { + "input_tokens": 125234, + "output_tokens": 1998, + "total_tokens": 127232, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 126.32447290420532, + "agent_execution_time": 126.32447290420532, + "token_usage": { + "input_tokens": 263845, + "output_tokens": 2706, + "total_tokens": 266551, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 60.407455921173096, + "agent_execution_time": 60.407455921173096, + "token_usage": { + "input_tokens": 136487, + "output_tokens": 1918, + "total_tokens": 138405, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 59.993492126464844, + "agent_execution_time": 59.993492126464844, + "token_usage": { + "input_tokens": 138206, + "output_tokens": 1862, + "total_tokens": 140068, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 89.51, + "avg_input_tokens": 165943.0, + "avg_output_tokens": 2121.0, + "avg_total_tokens": 168064.0, + "avg_turn_count": 12.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 117.18397498130798, + "agent_execution_time": 117.18397498130798, + "token_usage": { + "input_tokens": 133327, + "output_tokens": 1592, + "total_tokens": 134919, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.769605875015259, + "agent_execution_time": 10.769605875015259, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 118.1135323047638, + "agent_execution_time": 118.1135323047638, + "token_usage": { + "input_tokens": 124956, + "output_tokens": 1322, + "total_tokens": 126278, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 14.015326023101807, + "agent_execution_time": 14.015326023101807, + "token_usage": { + "input_tokens": 24033, + "output_tokens": 87, + "total_tokens": 24120, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 65.02, + "avg_input_tokens": 70579.0, + "avg_output_tokens": 750.2, + "avg_total_tokens": 71329.2, + "avg_turn_count": 3.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 39.48687195777893, + "agent_execution_time": 39.48687195777893, + "token_usage": { + "input_tokens": 93496, + "output_tokens": 2789, + "total_tokens": 96285, + "reasoning_tokens": 1867 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": true, + "execution_time": 33.43517184257507, + "agent_execution_time": 33.43517184257507, + "token_usage": { + "input_tokens": 90667, + "output_tokens": 2464, + "total_tokens": 93131, + "reasoning_tokens": 1554 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": true, + "execution_time": 39.33463096618652, + "agent_execution_time": 39.33463096618652, + "token_usage": { + "input_tokens": 78225, + "output_tokens": 2565, + "total_tokens": 80790, + "reasoning_tokens": 1676 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 38.12186312675476, + "agent_execution_time": 38.12186312675476, + "token_usage": { + "input_tokens": 78418, + "output_tokens": 2642, + "total_tokens": 81060, + "reasoning_tokens": 1701 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 37.59, + "avg_input_tokens": 85201.5, + "avg_output_tokens": 2615.0, + "avg_total_tokens": 87816.5, + "avg_turn_count": 5.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/mcpmark-cicd__pr_automation_workflow.json b/task_results/mcpmark-cicd__pr_automation_workflow.json index 321bcbe1..2900dc75 100644 --- a/task_results/mcpmark-cicd__pr_automation_workflow.json +++ b/task_results/mcpmark-cicd__pr_automation_workflow.json @@ -2,6 +2,73 @@ "task": "mcpmark-cicd__pr_automation_workflow", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 273.5244388580322, + "agent_execution_time": 273.5244388580322, + "token_usage": { + "input_tokens": 266175, + "output_tokens": 4311, + "total_tokens": 270486, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 417.759516954422, + "agent_execution_time": 417.759516954422, + "token_usage": { + "input_tokens": 108295, + "output_tokens": 8799, + "total_tokens": 117094, + "reasoning_tokens": 4313 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 880.5527496337891, + "agent_execution_time": 880.5527496337891, + "token_usage": { + "input_tokens": 196076, + "output_tokens": 18662, + "total_tokens": 214738, + "reasoning_tokens": 16183 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 690.0449638366699, + "agent_execution_time": 690.0449638366699, + "token_usage": { + "input_tokens": 127083, + "output_tokens": 20718, + "total_tokens": 147801, + "reasoning_tokens": 19357 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 565.47, + "avg_input_tokens": 174407.2, + "avg_output_tokens": 13122.5, + "avg_total_tokens": 187529.8, + "avg_turn_count": 7.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 420.4544110298157, - "agent_execution_time": 420.4544110298157, - "token_usage": { - "input_tokens": 117291, - "output_tokens": 9198, - "total_tokens": 126489, - "reasoning_tokens": 0 - }, - "turn_count": 6 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 420.45, - "avg_input_tokens": 117291.0, - "avg_output_tokens": 9198.0, - "avg_total_tokens": 126489.0, - "avg_turn_count": 6.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 399.36375999450684, + "agent_execution_time": 399.36375999450684, + "token_usage": { + "input_tokens": 889788, + "output_tokens": 24070, + "total_tokens": 913858, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": false, + "execution_time": 883.0230395793915, + "agent_execution_time": 883.0230395793915, + "token_usage": { + "input_tokens": 3024248, + "output_tokens": 50971, + "total_tokens": 3075219, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-3", + "success": false, + "execution_time": 436.3579466342926, + "agent_execution_time": 436.3579466342926, + "token_usage": { + "input_tokens": 952340, + "output_tokens": 28101, + "total_tokens": 980441, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 566.5929629802704, + "agent_execution_time": 566.5929629802704, + "token_usage": { + "input_tokens": 644382, + "output_tokens": 35768, + "total_tokens": 680150, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 571.33, + "avg_input_tokens": 1377689.5, + "avg_output_tokens": 34727.5, + "avg_total_tokens": 1412417.0, + "avg_turn_count": 42.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 128.08987975120544, + "agent_execution_time": 128.08987975120544, + "token_usage": { + "input_tokens": 173599, + "output_tokens": 3824, + "total_tokens": 177423, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 130.76748967170715, + "agent_execution_time": 130.76748967170715, + "token_usage": { + "input_tokens": 201366, + "output_tokens": 4673, + "total_tokens": 206039, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 140.8674259185791, + "agent_execution_time": 140.8674259185791, + "token_usage": { + "input_tokens": 251204, + "output_tokens": 4737, + "total_tokens": 255941, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 171.66862630844116, + "agent_execution_time": 171.66862630844116, + "token_usage": { + "input_tokens": 215372, + "output_tokens": 5907, + "total_tokens": 221279, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 142.85, + "avg_input_tokens": 210385.2, + "avg_output_tokens": 4785.2, + "avg_total_tokens": 215170.5, + "avg_turn_count": 13.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 287.7944805622101, + "agent_execution_time": 287.7944805622101, + "token_usage": { + "input_tokens": 246758, + "output_tokens": 4898, + "total_tokens": 251656, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 11.336401224136353, + "agent_execution_time": 11.336401224136353, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 17.65446400642395, + "agent_execution_time": 17.65446400642395, + "token_usage": { + "input_tokens": 24006, + "output_tokens": 92, + "total_tokens": 24098, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 155.7432405948639, + "agent_execution_time": 155.7432405948639, + "token_usage": { + "input_tokens": 133207, + "output_tokens": 3844, + "total_tokens": 137051, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 118.13, + "avg_input_tokens": 100992.8, + "avg_output_tokens": 2208.5, + "avg_total_tokens": 103201.2, + "avg_turn_count": 4.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 31.984736680984497, + "agent_execution_time": 31.984736680984497, + "token_usage": { + "input_tokens": 81540, + "output_tokens": 3720, + "total_tokens": 85260, + "reasoning_tokens": 2078 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 64.02203607559204, + "agent_execution_time": 64.02203607559204, + "token_usage": { + "input_tokens": 80728, + "output_tokens": 4008, + "total_tokens": 84736, + "reasoning_tokens": 2497 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 52.280176639556885, + "agent_execution_time": 52.280176639556885, + "token_usage": { + "input_tokens": 81378, + "output_tokens": 3641, + "total_tokens": 85019, + "reasoning_tokens": 1916 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 56.474762201309204, + "agent_execution_time": 56.474762201309204, + "token_usage": { + "input_tokens": 81605, + "output_tokens": 3646, + "total_tokens": 85251, + "reasoning_tokens": 2448 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 51.19, + "avg_input_tokens": 81312.8, + "avg_output_tokens": 3753.8, + "avg_total_tokens": 85066.5, + "avg_turn_count": 5.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/missing-semester__assign_contributor_labels.json b/task_results/missing-semester__assign_contributor_labels.json index 9991b814..0df59100 100644 --- a/task_results/missing-semester__assign_contributor_labels.json +++ b/task_results/missing-semester__assign_contributor_labels.json @@ -2,6 +2,73 @@ "task": "missing-semester__assign_contributor_labels", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 253.2252287864685, + "agent_execution_time": 253.2252287864685, + "token_usage": { + "input_tokens": 1266459, + "output_tokens": 1772, + "total_tokens": 1268231, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 2348.931410074234, + "agent_execution_time": 2348.931410074234, + "token_usage": { + "input_tokens": 510937, + "output_tokens": 51406, + "total_tokens": 562343, + "reasoning_tokens": 50324 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 2197.3706357479095, + "agent_execution_time": 2197.3706357479095, + "token_usage": { + "input_tokens": 393220, + "output_tokens": 47749, + "total_tokens": 440969, + "reasoning_tokens": 45008 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 2324.812409877777, + "agent_execution_time": 2324.812409877777, + "token_usage": { + "input_tokens": 548256, + "output_tokens": 45120, + "total_tokens": 593376, + "reasoning_tokens": 42697 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1781.08, + "avg_input_tokens": 679718.0, + "avg_output_tokens": 36511.8, + "avg_total_tokens": 716229.8, + "avg_turn_count": 10.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 110.90351128578186, - "agent_execution_time": 110.90351128578186, - "token_usage": { - "input_tokens": 370537, - "output_tokens": 829, - "total_tokens": 371366, - "reasoning_tokens": null - }, - "turn_count": 3 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 110.9, - "avg_input_tokens": 370537.0, - "avg_output_tokens": 829.0, - "avg_total_tokens": 371366.0, - "avg_turn_count": 3.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 108.81156086921692, + "agent_execution_time": 108.81156086921692, + "token_usage": { + "input_tokens": 577979, + "output_tokens": 4005, + "total_tokens": 581984, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 69.24230194091797, + "agent_execution_time": 69.24230194091797, + "token_usage": { + "input_tokens": 278352, + "output_tokens": 2642, + "total_tokens": 280994, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 97.96777606010437, + "agent_execution_time": 97.96777606010437, + "token_usage": { + "input_tokens": 523025, + "output_tokens": 3554, + "total_tokens": 526579, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 83.05909609794617, + "agent_execution_time": 83.05909609794617, + "token_usage": { + "input_tokens": 378019, + "output_tokens": 3102, + "total_tokens": 381121, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 89.77, + "avg_input_tokens": 439343.8, + "avg_output_tokens": 3325.8, + "avg_total_tokens": 442669.5, + "avg_turn_count": 5.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 191.44082021713257, + "agent_execution_time": 191.44082021713257, + "token_usage": { + "input_tokens": 1423872, + "output_tokens": 2043, + "total_tokens": 1425915, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": false, + "execution_time": 120.27857232093811, + "agent_execution_time": 120.27857232093811, + "token_usage": { + "input_tokens": 1040497, + "output_tokens": 1938, + "total_tokens": 1042435, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 130.93763709068298, + "agent_execution_time": 130.93763709068298, + "token_usage": { + "input_tokens": 1031150, + "output_tokens": 1733, + "total_tokens": 1032883, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 114.22383570671082, + "agent_execution_time": 114.22383570671082, + "token_usage": { + "input_tokens": 974702, + "output_tokens": 1975, + "total_tokens": 976677, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 139.22, + "avg_input_tokens": 1117555.2, + "avg_output_tokens": 1922.2, + "avg_total_tokens": 1119477.5, + "avg_turn_count": 19.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 235.4257414340973, + "agent_execution_time": 235.4257414340973, + "token_usage": { + "input_tokens": 1264610, + "output_tokens": 1617, + "total_tokens": 1266227, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 11.687310695648193, + "agent_execution_time": 11.687310695648193, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 613.6650176048279, + "agent_execution_time": 613.6650176048279, + "token_usage": { + "input_tokens": 1047892, + "output_tokens": 2133, + "total_tokens": 1050025, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 151.15459322929382, + "agent_execution_time": 151.15459322929382, + "token_usage": { + "input_tokens": 380809, + "output_tokens": 1588, + "total_tokens": 382397, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 252.98, + "avg_input_tokens": 673327.8, + "avg_output_tokens": 1334.5, + "avg_total_tokens": 674662.2, + "avg_turn_count": 9.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 55.51180863380432, + "agent_execution_time": 55.51180863380432, + "token_usage": { + "input_tokens": 165074, + "output_tokens": 4050, + "total_tokens": 169124, + "reasoning_tokens": 2929 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 48.97187948226929, + "agent_execution_time": 48.97187948226929, + "token_usage": { + "input_tokens": 150499, + "output_tokens": 4446, + "total_tokens": 154945, + "reasoning_tokens": 3891 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 69.35130429267883, + "agent_execution_time": 69.35130429267883, + "token_usage": { + "input_tokens": 188252, + "output_tokens": 5373, + "total_tokens": 193625, + "reasoning_tokens": 3764 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 64.27985310554504, + "agent_execution_time": 64.27985310554504, + "token_usage": { + "input_tokens": 217762, + "output_tokens": 4888, + "total_tokens": 222650, + "reasoning_tokens": 3376 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 59.53, + "avg_input_tokens": 180396.8, + "avg_output_tokens": 4689.2, + "avg_total_tokens": 185086.0, + "avg_turn_count": 6.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/missing-semester__find_legacy_name.json b/task_results/missing-semester__find_legacy_name.json index d8a1144f..3612e55e 100644 --- a/task_results/missing-semester__find_legacy_name.json +++ b/task_results/missing-semester__find_legacy_name.json @@ -2,6 +2,73 @@ "task": "missing-semester__find_legacy_name", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 253.85763430595398, + "agent_execution_time": 253.85763430595398, + "token_usage": { + "input_tokens": 1212872, + "output_tokens": 878, + "total_tokens": 1213750, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 488.5240752696991, + "agent_execution_time": 488.5240752696991, + "token_usage": { + "input_tokens": 246380, + "output_tokens": 9651, + "total_tokens": 256031, + "reasoning_tokens": 9417 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 689.0475895404816, + "agent_execution_time": 689.0475895404816, + "token_usage": { + "input_tokens": 161943, + "output_tokens": 7204, + "total_tokens": 169147, + "reasoning_tokens": 6986 + }, + "turn_count": 3 + }, + { + "run": "run-4", + "success": false, + "execution_time": 195.33893036842346, + "agent_execution_time": 195.33893036842346, + "token_usage": { + "input_tokens": 132351, + "output_tokens": 4721, + "total_tokens": 137072, + "reasoning_tokens": 4562 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 406.69, + "avg_input_tokens": 438386.5, + "avg_output_tokens": 5613.5, + "avg_total_tokens": 444000.0, + "avg_turn_count": 6.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 58.568087577819824, - "agent_execution_time": 58.568087577819824, - "token_usage": { - "input_tokens": 192528, - "output_tokens": 290, - "total_tokens": 192818, - "reasoning_tokens": null - }, - "turn_count": 2 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 58.57, - "avg_input_tokens": 192528.0, - "avg_output_tokens": 290.0, - "avg_total_tokens": 192818.0, - "avg_turn_count": 2.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 98.60856485366821, + "agent_execution_time": 98.60856485366821, + "token_usage": { + "input_tokens": 1275937, + "output_tokens": 1705, + "total_tokens": 1277642, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": true, + "execution_time": 109.07105731964111, + "agent_execution_time": 109.07105731964111, + "token_usage": { + "input_tokens": 1349140, + "output_tokens": 2386, + "total_tokens": 1351526, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": true, + "execution_time": 153.1866273880005, + "agent_execution_time": 153.1866273880005, + "token_usage": { + "input_tokens": 1860761, + "output_tokens": 2830, + "total_tokens": 1863591, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": true, + "execution_time": 101.16169309616089, + "agent_execution_time": 101.16169309616089, + "token_usage": { + "input_tokens": 1345107, + "output_tokens": 1906, + "total_tokens": 1347013, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 115.51, + "avg_input_tokens": 1457736.2, + "avg_output_tokens": 2206.8, + "avg_total_tokens": 1459943.0, + "avg_turn_count": 14.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 129.23458671569824, + "agent_execution_time": 129.23458671569824, + "token_usage": { + "input_tokens": 641886, + "output_tokens": 884, + "total_tokens": 642770, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 144.26113295555115, + "agent_execution_time": 144.26113295555115, + "token_usage": { + "input_tokens": 697641, + "output_tokens": 923, + "total_tokens": 698564, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 232.479074716568, + "agent_execution_time": 232.479074716568, + "token_usage": { + "input_tokens": 1334810, + "output_tokens": 1142, + "total_tokens": 1335952, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 190.98346781730652, + "agent_execution_time": 190.98346781730652, + "token_usage": { + "input_tokens": 706977, + "output_tokens": 893, + "total_tokens": 707870, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 174.24, + "avg_input_tokens": 845328.5, + "avg_output_tokens": 960.5, + "avg_total_tokens": 846289.0, + "avg_turn_count": 11.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 93.55011773109436, + "agent_execution_time": 93.55011773109436, + "token_usage": { + "input_tokens": 247911, + "output_tokens": 263, + "total_tokens": 248174, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": false, + "execution_time": 10.70331597328186, + "agent_execution_time": 10.70331597328186, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 744.953742980957, + "agent_execution_time": 744.953742980957, + "token_usage": { + "input_tokens": 1089926, + "output_tokens": 790, + "total_tokens": 1090716, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 118.13037371635437, + "agent_execution_time": 118.13037371635437, + "token_usage": { + "input_tokens": 331872, + "output_tokens": 489, + "total_tokens": 332361, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 241.83, + "avg_input_tokens": 417427.2, + "avg_output_tokens": 385.5, + "avg_total_tokens": 417812.8, + "avg_turn_count": 5.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 97.89311742782593, + "agent_execution_time": 97.89311742782593, + "token_usage": { + "input_tokens": 263734, + "output_tokens": 8011, + "total_tokens": 271745, + "reasoning_tokens": 2588 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 72.64546275138855, + "agent_execution_time": 72.64546275138855, + "token_usage": { + "input_tokens": 123187, + "output_tokens": 6000, + "total_tokens": 129187, + "reasoning_tokens": 5639 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 122.08328175544739, + "agent_execution_time": 122.08328175544739, + "token_usage": { + "input_tokens": 719413, + "output_tokens": 8831, + "total_tokens": 728244, + "reasoning_tokens": 5668 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 112.49965167045593, + "agent_execution_time": 112.49965167045593, + "token_usage": { + "input_tokens": 633035, + "output_tokens": 7615, + "total_tokens": 640650, + "reasoning_tokens": 3994 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 101.28, + "avg_input_tokens": 434842.2, + "avg_output_tokens": 7614.2, + "avg_total_tokens": 442456.5, + "avg_turn_count": 11.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/missing-semester__find_salient_file.json b/task_results/missing-semester__find_salient_file.json index 4506fb54..aaa4a424 100644 --- a/task_results/missing-semester__find_salient_file.json +++ b/task_results/missing-semester__find_salient_file.json @@ -2,6 +2,73 @@ "task": "missing-semester__find_salient_file", "service": "github", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 389.75921511650085, + "agent_execution_time": 389.75921511650085, + "token_usage": { + "input_tokens": 1782919, + "output_tokens": 3457, + "total_tokens": 1786376, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + { + "run": "run-2", + "success": false, + "execution_time": 649.1462063789368, + "agent_execution_time": 649.1462063789368, + "token_usage": { + "input_tokens": 101856, + "output_tokens": 15399, + "total_tokens": 117255, + "reasoning_tokens": 11191 + }, + "turn_count": 3 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1574.7181437015533, + "agent_execution_time": 1574.7181437015533, + "token_usage": { + "input_tokens": 435629, + "output_tokens": 35580, + "total_tokens": 471209, + "reasoning_tokens": 32311 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1646.8885293006897, + "agent_execution_time": 1646.8885293006897, + "token_usage": { + "input_tokens": 423388, + "output_tokens": 44367, + "total_tokens": 467755, + "reasoning_tokens": 38187 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1065.13, + "avg_input_tokens": 685948.0, + "avg_output_tokens": 24700.8, + "avg_total_tokens": 710648.8, + "avg_turn_count": 12.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 321.5846378803253, - "agent_execution_time": 321.5846378803253, - "token_usage": { - "input_tokens": 1509742, - "output_tokens": 1380, - "total_tokens": 1511122, - "reasoning_tokens": null - }, - "turn_count": 9 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 321.58, - "avg_input_tokens": 1509742.0, - "avg_output_tokens": 1380.0, - "avg_total_tokens": 1511122.0, - "avg_turn_count": 9.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 372.2928993701935, + "agent_execution_time": 372.2928993701935, + "token_usage": { + "input_tokens": 1879587, + "output_tokens": 11574, + "total_tokens": 1891161, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": false, + "execution_time": 139.31281876564026, + "agent_execution_time": 139.31281876564026, + "token_usage": { + "input_tokens": 1059119, + "output_tokens": 3923, + "total_tokens": 1063042, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 306.26760482788086, + "agent_execution_time": 306.26760482788086, + "token_usage": { + "input_tokens": 1449960, + "output_tokens": 9016, + "total_tokens": 1458976, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": true, + "execution_time": 389.31269216537476, + "agent_execution_time": 389.31269216537476, + "token_usage": { + "input_tokens": 1915652, + "output_tokens": 12221, + "total_tokens": 1927873, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 301.8, + "avg_input_tokens": 1576079.5, + "avg_output_tokens": 9183.5, + "avg_total_tokens": 1585263.0, + "avg_turn_count": 19.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 647.3342833518982, + "agent_execution_time": 647.3342833518982, + "token_usage": { + "input_tokens": 4826411, + "output_tokens": 6408, + "total_tokens": 4832819, + "reasoning_tokens": 0 + }, + "turn_count": 86 + }, + { + "run": "run-2", + "success": false, + "execution_time": 451.8009171485901, + "agent_execution_time": 451.8009171485901, + "token_usage": { + "input_tokens": 4751937, + "output_tokens": 6311, + "total_tokens": 4758248, + "reasoning_tokens": 0 + }, + "turn_count": 85 + }, + { + "run": "run-3", + "success": false, + "execution_time": 229.7355453968048, + "agent_execution_time": 229.7355453968048, + "token_usage": { + "input_tokens": 1697498, + "output_tokens": 5524, + "total_tokens": 1703022, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + { + "run": "run-4", + "success": false, + "execution_time": 402.2834470272064, + "agent_execution_time": 402.2834470272064, + "token_usage": { + "input_tokens": 4168805, + "output_tokens": 5807, + "total_tokens": 4174612, + "reasoning_tokens": 0 + }, + "turn_count": 77 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 432.79, + "avg_input_tokens": 3861162.8, + "avg_output_tokens": 6012.5, + "avg_total_tokens": 3867175.2, + "avg_turn_count": 71.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 560.5716016292572, + "agent_execution_time": 560.5716016292572, + "token_usage": { + "input_tokens": 2938725, + "output_tokens": 5036, + "total_tokens": 2943761, + "reasoning_tokens": 0 + }, + "turn_count": 50 + }, + { + "run": "run-2", + "success": false, + "execution_time": 11.157562017440796, + "agent_execution_time": 11.157562017440796, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + }, + { + "run": "run-3", + "success": false, + "execution_time": 902.2717890739441, + "agent_execution_time": 902.2717890739441, + "token_usage": { + "input_tokens": 1524753, + "output_tokens": 4662, + "total_tokens": 1529415, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": false, + "execution_time": 411.1900761127472, + "agent_execution_time": 411.1900761127472, + "token_usage": { + "input_tokens": 1171924, + "output_tokens": 3430, + "total_tokens": 1175354, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 471.3, + "avg_input_tokens": 1408850.5, + "avg_output_tokens": 3282.0, + "avg_total_tokens": 1412132.5, + "avg_turn_count": 23.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 114.99301266670227, + "agent_execution_time": 114.99301266670227, + "token_usage": { + "input_tokens": 1085121, + "output_tokens": 7416, + "total_tokens": 1092537, + "reasoning_tokens": 5868 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 85.67410707473755, + "agent_execution_time": 85.67410707473755, + "token_usage": { + "input_tokens": 360696, + "output_tokens": 6896, + "total_tokens": 367592, + "reasoning_tokens": 4442 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 108.95954179763794, + "agent_execution_time": 108.95954179763794, + "token_usage": { + "input_tokens": 425883, + "output_tokens": 7102, + "total_tokens": 432985, + "reasoning_tokens": 4500 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 38.620139837265015, + "agent_execution_time": 38.620139837265015, + "token_usage": { + "input_tokens": 108435, + "output_tokens": 3056, + "total_tokens": 111491, + "reasoning_tokens": 2599 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 87.06, + "avg_input_tokens": 495033.8, + "avg_output_tokens": 6117.5, + "avg_total_tokens": 501151.2, + "avg_turn_count": 11.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/online_resume__layout_adjustment.json b/task_results/online_resume__layout_adjustment.json index f991b550..5b1a011d 100644 --- a/task_results/online_resume__layout_adjustment.json +++ b/task_results/online_resume__layout_adjustment.json @@ -2,6 +2,73 @@ "task": "online_resume__layout_adjustment", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 399.45585775375366, + "agent_execution_time": 399.45585775375366, + "token_usage": { + "input_tokens": 1347645, + "output_tokens": 3756, + "total_tokens": 1351401, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + { + "run": "run-2", + "success": false, + "execution_time": 795.1876745223999, + "agent_execution_time": 795.1876745223999, + "token_usage": { + "input_tokens": 154670, + "output_tokens": 19477, + "total_tokens": 174147, + "reasoning_tokens": 17564 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 343.291757106781, + "agent_execution_time": 343.291757106781, + "token_usage": { + "input_tokens": 1196089, + "output_tokens": 3083, + "total_tokens": 1199172, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-4", + "success": false, + "execution_time": 2117.443124771118, + "agent_execution_time": 2117.443124771118, + "token_usage": { + "input_tokens": 416183, + "output_tokens": 42671, + "total_tokens": 458854, + "reasoning_tokens": 39603 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 913.84, + "avg_input_tokens": 778646.8, + "avg_output_tokens": 17246.8, + "avg_total_tokens": 795893.5, + "avg_turn_count": 24.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 425.41325330734253, - "agent_execution_time": 425.41325330734253, - "token_usage": { - "input_tokens": 3192034, - "output_tokens": 6401, - "total_tokens": 3198435, - "reasoning_tokens": null - }, - "turn_count": 40 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 425.41, - "avg_input_tokens": 3192034.0, - "avg_output_tokens": 6401.0, - "avg_total_tokens": 3198435.0, - "avg_turn_count": 40.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 148.5750207901001, + "agent_execution_time": 148.5750207901001, + "token_usage": { + "input_tokens": 552532, + "output_tokens": 4933, + "total_tokens": 557465, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 157.71774888038635, + "agent_execution_time": 157.71774888038635, + "token_usage": { + "input_tokens": 683061, + "output_tokens": 5658, + "total_tokens": 688719, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 190.4845688343048, + "agent_execution_time": 190.4845688343048, + "token_usage": { + "input_tokens": 799547, + "output_tokens": 5602, + "total_tokens": 805149, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 165.11123919487, + "agent_execution_time": 165.11123919487, + "token_usage": { + "input_tokens": 595091, + "output_tokens": 4789, + "total_tokens": 599880, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 165.47, + "avg_input_tokens": 657557.8, + "avg_output_tokens": 5245.5, + "avg_total_tokens": 662803.2, + "avg_turn_count": 18.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 173.49288392066956, + "agent_execution_time": 173.49288392066956, + "token_usage": { + "input_tokens": 904879, + "output_tokens": 2968, + "total_tokens": 907847, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + { + "run": "run-2", + "success": false, + "execution_time": 122.97894787788391, + "agent_execution_time": 122.97894787788391, + "token_usage": { + "input_tokens": 647305, + "output_tokens": 2118, + "total_tokens": 649423, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": false, + "execution_time": 321.3583390712738, + "agent_execution_time": 321.3583390712738, + "token_usage": { + "input_tokens": 1118278, + "output_tokens": 4761, + "total_tokens": 1123039, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + { + "run": "run-4", + "success": false, + "execution_time": 260.9174680709839, + "agent_execution_time": 260.9174680709839, + "token_usage": { + "input_tokens": 1057505, + "output_tokens": 5652, + "total_tokens": 1063157, + "reasoning_tokens": 0 + }, + "turn_count": 34 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 219.69, + "avg_input_tokens": 931991.8, + "avg_output_tokens": 3874.8, + "avg_total_tokens": 935866.5, + "avg_turn_count": 31.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 280.65002727508545, + "agent_execution_time": 280.65002727508545, + "token_usage": { + "input_tokens": 821187, + "output_tokens": 2830, + "total_tokens": 824017, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 394.77181601524353, + "agent_execution_time": 394.77181601524353, + "token_usage": { + "input_tokens": 1027774, + "output_tokens": 3167, + "total_tokens": 1030941, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-3", + "success": false, + "execution_time": 272.48287296295166, + "agent_execution_time": 272.48287296295166, + "token_usage": { + "input_tokens": 836279, + "output_tokens": 2693, + "total_tokens": 838972, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-4", + "success": false, + "execution_time": 310.2703666687012, + "agent_execution_time": 310.2703666687012, + "token_usage": { + "input_tokens": 980604, + "output_tokens": 2752, + "total_tokens": 983356, + "reasoning_tokens": 0 + }, + "turn_count": 29 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 314.54, + "avg_input_tokens": 916461.0, + "avg_output_tokens": 2860.5, + "avg_total_tokens": 919321.5, + "avg_turn_count": 28.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 97.22220754623413, + "agent_execution_time": 97.22220754623413, + "token_usage": { + "input_tokens": 436107, + "output_tokens": 8564, + "total_tokens": 444671, + "reasoning_tokens": 5824 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 72.72354555130005, + "agent_execution_time": 72.72354555130005, + "token_usage": { + "input_tokens": 466141, + "output_tokens": 6985, + "total_tokens": 473126, + "reasoning_tokens": 5731 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": false, + "execution_time": 13.191792964935303, + "agent_execution_time": 13.191792964935303, + "token_usage": { + "input_tokens": 18362, + "output_tokens": 1037, + "total_tokens": 19399, + "reasoning_tokens": 921 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 148.71358561515808, + "agent_execution_time": 148.71358561515808, + "token_usage": { + "input_tokens": 1547591, + "output_tokens": 7623, + "total_tokens": 1555214, + "reasoning_tokens": 5551 + }, + "turn_count": 36 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 82.96, + "avg_input_tokens": 617050.2, + "avg_output_tokens": 6052.2, + "avg_total_tokens": 623102.5, + "avg_turn_count": 19.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/online_resume__projects_section_update.json b/task_results/online_resume__projects_section_update.json index 2424ec82..6cb7e09c 100644 --- a/task_results/online_resume__projects_section_update.json +++ b/task_results/online_resume__projects_section_update.json @@ -2,6 +2,73 @@ "task": "online_resume__projects_section_update", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 242.21766686439514, + "agent_execution_time": 242.21766686439514, + "token_usage": { + "input_tokens": 699039, + "output_tokens": 2334, + "total_tokens": 701373, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": true, + "execution_time": 1428.3427770137787, + "agent_execution_time": 1428.3427770137787, + "token_usage": { + "input_tokens": 267372, + "output_tokens": 32971, + "total_tokens": 300343, + "reasoning_tokens": 31280 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 214.03604531288147, + "agent_execution_time": 214.03604531288147, + "token_usage": { + "input_tokens": 567973, + "output_tokens": 2085, + "total_tokens": 570058, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1910.665857553482, + "agent_execution_time": 1910.665857553482, + "token_usage": { + "input_tokens": 283888, + "output_tokens": 41540, + "total_tokens": 325428, + "reasoning_tokens": 39272 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 948.82, + "avg_input_tokens": 454568.0, + "avg_output_tokens": 19732.5, + "avg_total_tokens": 474300.5, + "avg_turn_count": 17.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 181.69771671295166, - "agent_execution_time": 181.69771671295166, - "token_usage": { - "input_tokens": 847570, - "output_tokens": 2903, - "total_tokens": 850473, - "reasoning_tokens": null - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 181.7, - "avg_input_tokens": 847570.0, - "avg_output_tokens": 2903.0, - "avg_total_tokens": 850473.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 82.00306606292725, + "agent_execution_time": 82.00306606292725, + "token_usage": { + "input_tokens": 308566, + "output_tokens": 2697, + "total_tokens": 311263, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 119.67876672744751, + "agent_execution_time": 119.67876672744751, + "token_usage": { + "input_tokens": 332267, + "output_tokens": 3807, + "total_tokens": 336074, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": true, + "execution_time": 194.65828442573547, + "agent_execution_time": 194.65828442573547, + "token_usage": { + "input_tokens": 554726, + "output_tokens": 6195, + "total_tokens": 560921, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 212.98318338394165, + "agent_execution_time": 212.98318338394165, + "token_usage": { + "input_tokens": 1334677, + "output_tokens": 4448, + "total_tokens": 1339125, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 152.33, + "avg_input_tokens": 632559.0, + "avg_output_tokens": 4286.8, + "avg_total_tokens": 636845.8, + "avg_turn_count": 15.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 176.80517601966858, + "agent_execution_time": 176.80517601966858, + "token_usage": { + "input_tokens": 723756, + "output_tokens": 2837, + "total_tokens": 726593, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-2", + "success": true, + "execution_time": 98.39138841629028, + "agent_execution_time": 98.39138841629028, + "token_usage": { + "input_tokens": 409236, + "output_tokens": 1843, + "total_tokens": 411079, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 250.32507753372192, + "agent_execution_time": 250.32507753372192, + "token_usage": { + "input_tokens": 343554, + "output_tokens": 3333, + "total_tokens": 346887, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": true, + "execution_time": 200.62014293670654, + "agent_execution_time": 200.62014293670654, + "token_usage": { + "input_tokens": 898213, + "output_tokens": 3919, + "total_tokens": 902132, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 181.54, + "avg_input_tokens": 593689.8, + "avg_output_tokens": 2983.0, + "avg_total_tokens": 596672.8, + "avg_turn_count": 22.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 240.42282342910767, + "agent_execution_time": 240.42282342910767, + "token_usage": { + "input_tokens": 679571, + "output_tokens": 2464, + "total_tokens": 682035, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 95.56528234481812, + "agent_execution_time": 95.56528234481812, + "token_usage": { + "input_tokens": 226274, + "output_tokens": 929, + "total_tokens": 227203, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": true, + "execution_time": 235.71794629096985, + "agent_execution_time": 235.71794629096985, + "token_usage": { + "input_tokens": 629808, + "output_tokens": 2408, + "total_tokens": 632216, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 305.6466438770294, + "agent_execution_time": 305.6466438770294, + "token_usage": { + "input_tokens": 914592, + "output_tokens": 2838, + "total_tokens": 917430, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 219.34, + "avg_input_tokens": 612561.2, + "avg_output_tokens": 2159.8, + "avg_total_tokens": 614721.0, + "avg_turn_count": 19.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 100.46130013465881, + "agent_execution_time": 100.46130013465881, + "token_usage": { + "input_tokens": 602996, + "output_tokens": 7302, + "total_tokens": 610298, + "reasoning_tokens": 5272 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 97.22893905639648, + "agent_execution_time": 97.22893905639648, + "token_usage": { + "input_tokens": 768191, + "output_tokens": 7471, + "total_tokens": 775662, + "reasoning_tokens": 5080 + }, + "turn_count": 32 + }, + { + "run": "run-3", + "success": false, + "execution_time": 64.34393429756165, + "agent_execution_time": 64.34393429756165, + "token_usage": { + "input_tokens": 283142, + "output_tokens": 5006, + "total_tokens": 288148, + "reasoning_tokens": 3752 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 110.76311779022217, + "agent_execution_time": 110.76311779022217, + "token_usage": { + "input_tokens": 520703, + "output_tokens": 7341, + "total_tokens": 528044, + "reasoning_tokens": 4877 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 93.2, + "avg_input_tokens": 543758.0, + "avg_output_tokens": 6780.0, + "avg_total_tokens": 550538.0, + "avg_turn_count": 25.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/online_resume__skills_development_tracker.json b/task_results/online_resume__skills_development_tracker.json index 76bb82e6..34222de0 100644 --- a/task_results/online_resume__skills_development_tracker.json +++ b/task_results/online_resume__skills_development_tracker.json @@ -2,6 +2,73 @@ "task": "online_resume__skills_development_tracker", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 349.7716019153595, + "agent_execution_time": 349.7716019153595, + "token_usage": { + "input_tokens": 733127, + "output_tokens": 4388, + "total_tokens": 737515, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-2", + "success": false, + "execution_time": 2725.222920656204, + "agent_execution_time": 2725.222920656204, + "token_usage": { + "input_tokens": 362322, + "output_tokens": 62964, + "total_tokens": 425286, + "reasoning_tokens": 56896 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": true, + "execution_time": 549.9365866184235, + "agent_execution_time": 549.9365866184235, + "token_usage": { + "input_tokens": 1559618, + "output_tokens": 5850, + "total_tokens": 1565468, + "reasoning_tokens": 0 + }, + "turn_count": 46 + }, + { + "run": "run-4", + "success": false, + "execution_time": 3084.928017139435, + "agent_execution_time": 3084.928017139435, + "token_usage": { + "input_tokens": 233650, + "output_tokens": 63320, + "total_tokens": 296970, + "reasoning_tokens": 57188 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 1677.46, + "avg_input_tokens": 722179.2, + "avg_output_tokens": 34130.5, + "avg_total_tokens": 756309.8, + "avg_turn_count": 26.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 247.4410319328308, - "agent_execution_time": 247.4410319328308, - "token_usage": { - "input_tokens": 622238, - "output_tokens": 4324, - "total_tokens": 626562, - "reasoning_tokens": null - }, - "turn_count": 19 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 247.44, - "avg_input_tokens": 622238.0, - "avg_output_tokens": 4324.0, - "avg_total_tokens": 626562.0, - "avg_turn_count": 19.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 259.40184354782104, + "agent_execution_time": 259.40184354782104, + "token_usage": { + "input_tokens": 729375, + "output_tokens": 7270, + "total_tokens": 736645, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": false, + "execution_time": 118.78692293167114, + "agent_execution_time": 118.78692293167114, + "token_usage": { + "input_tokens": 205288, + "output_tokens": 3679, + "total_tokens": 208967, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 134.834406375885, + "agent_execution_time": 134.834406375885, + "token_usage": { + "input_tokens": 527634, + "output_tokens": 4964, + "total_tokens": 532598, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": true, + "execution_time": 300.42046642303467, + "agent_execution_time": 300.42046642303467, + "token_usage": { + "input_tokens": 1284658, + "output_tokens": 5675, + "total_tokens": 1290333, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 203.36, + "avg_input_tokens": 686738.8, + "avg_output_tokens": 5397.0, + "avg_total_tokens": 692135.8, + "avg_turn_count": 17.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 1498.7769165039062, + "agent_execution_time": 1498.7769165039062, + "token_usage": { + "input_tokens": 6542043, + "output_tokens": 24570, + "total_tokens": 6566613, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-2", + "success": false, + "execution_time": 218.85236644744873, + "agent_execution_time": 218.85236644744873, + "token_usage": { + "input_tokens": 689645, + "output_tokens": 5092, + "total_tokens": 694737, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-3", + "success": false, + "execution_time": 386.03705072402954, + "agent_execution_time": 386.03705072402954, + "token_usage": { + "input_tokens": 945226, + "output_tokens": 4751, + "total_tokens": 949977, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": false, + "execution_time": 180.2046902179718, + "agent_execution_time": 180.2046902179718, + "token_usage": { + "input_tokens": 549953, + "output_tokens": 4914, + "total_tokens": 554867, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 570.97, + "avg_input_tokens": 2181716.8, + "avg_output_tokens": 9831.8, + "avg_total_tokens": 2191548.5, + "avg_turn_count": 45.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 370.6589341163635, + "agent_execution_time": 370.6589341163635, + "token_usage": { + "input_tokens": 968800, + "output_tokens": 4270, + "total_tokens": 973070, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-2", + "success": false, + "execution_time": 11.19611644744873, + "agent_execution_time": 11.19611644744873, + "token_usage": { + "input_tokens": 6710, + "output_tokens": 215, + "total_tokens": 6925, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": false, + "execution_time": 559.2584075927734, + "agent_execution_time": 559.2584075927734, + "token_usage": { + "input_tokens": 2200059, + "output_tokens": 5726, + "total_tokens": 2205785, + "reasoning_tokens": 0 + }, + "turn_count": 48 + }, + { + "run": "run-4", + "success": false, + "execution_time": 295.5869219303131, + "agent_execution_time": 295.5869219303131, + "token_usage": { + "input_tokens": 424892, + "output_tokens": 3350, + "total_tokens": 428242, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 309.18, + "avg_input_tokens": 900115.2, + "avg_output_tokens": 3390.2, + "avg_total_tokens": 903505.5, + "avg_turn_count": 26.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 148.39083409309387, + "agent_execution_time": 148.39083409309387, + "token_usage": { + "input_tokens": 606963, + "output_tokens": 11980, + "total_tokens": 618943, + "reasoning_tokens": 8702 + }, + "turn_count": 33 + }, + { + "run": "run-2", + "success": false, + "execution_time": 92.16241908073425, + "agent_execution_time": 92.16241908073425, + "token_usage": { + "input_tokens": 216474, + "output_tokens": 9529, + "total_tokens": 226003, + "reasoning_tokens": 7334 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 103.72020506858826, + "agent_execution_time": 103.72020506858826, + "token_usage": { + "input_tokens": 415296, + "output_tokens": 9315, + "total_tokens": 424611, + "reasoning_tokens": 7033 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 41.46541929244995, + "agent_execution_time": 41.46541929244995, + "token_usage": { + "input_tokens": 22037, + "output_tokens": 3834, + "total_tokens": 25871, + "reasoning_tokens": 2454 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 96.43, + "avg_input_tokens": 315192.5, + "avg_output_tokens": 8664.5, + "avg_total_tokens": 323857.0, + "avg_turn_count": 18.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/online_resume__work_history_addition.json b/task_results/online_resume__work_history_addition.json index a2a50f62..78cf49cd 100644 --- a/task_results/online_resume__work_history_addition.json +++ b/task_results/online_resume__work_history_addition.json @@ -2,6 +2,73 @@ "task": "online_resume__work_history_addition", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 372.8326954841614, + "agent_execution_time": 372.8326954841614, + "token_usage": { + "input_tokens": 551759, + "output_tokens": 5426, + "total_tokens": 557185, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1150.1524181365967, + "agent_execution_time": 1150.1524181365967, + "token_usage": { + "input_tokens": 154643, + "output_tokens": 27462, + "total_tokens": 182105, + "reasoning_tokens": 26398 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 201.71948194503784, + "agent_execution_time": 201.71948194503784, + "token_usage": { + "input_tokens": 486528, + "output_tokens": 2340, + "total_tokens": 488868, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 268.417772769928, + "agent_execution_time": 268.417772769928, + "token_usage": { + "input_tokens": 6589, + "output_tokens": 6150, + "total_tokens": 12739, + "reasoning_tokens": 6082 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 498.28, + "avg_input_tokens": 299879.8, + "avg_output_tokens": 10344.5, + "avg_total_tokens": 310224.2, + "avg_turn_count": 13.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 211.4523344039917, - "agent_execution_time": 211.4523344039917, - "token_usage": { - "input_tokens": 438871, - "output_tokens": 3223, - "total_tokens": 442094, - "reasoning_tokens": null - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 211.45, - "avg_input_tokens": 438871.0, - "avg_output_tokens": 3223.0, - "avg_total_tokens": 442094.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 112.69288372993469, + "agent_execution_time": 112.69288372993469, + "token_usage": { + "input_tokens": 322254, + "output_tokens": 3549, + "total_tokens": 325803, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 130.8178744316101, + "agent_execution_time": 130.8178744316101, + "token_usage": { + "input_tokens": 441689, + "output_tokens": 4362, + "total_tokens": 446051, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 102.33282542228699, + "agent_execution_time": 102.33282542228699, + "token_usage": { + "input_tokens": 178051, + "output_tokens": 2601, + "total_tokens": 180652, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 118.62417793273926, + "agent_execution_time": 118.62417793273926, + "token_usage": { + "input_tokens": 662244, + "output_tokens": 3312, + "total_tokens": 665556, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 116.12, + "avg_input_tokens": 401059.5, + "avg_output_tokens": 3456.0, + "avg_total_tokens": 404515.5, + "avg_turn_count": 13.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 133.6851305961609, + "agent_execution_time": 133.6851305961609, + "token_usage": { + "input_tokens": 282638, + "output_tokens": 3660, + "total_tokens": 286298, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 878.5687198638916, + "agent_execution_time": 878.5687198638916, + "token_usage": { + "input_tokens": 1089743, + "output_tokens": 31444, + "total_tokens": 1121187, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + { + "run": "run-3", + "success": false, + "execution_time": 325.2312822341919, + "agent_execution_time": 325.2312822341919, + "token_usage": { + "input_tokens": 588820, + "output_tokens": 2185, + "total_tokens": 591005, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 262.51826524734497, + "agent_execution_time": 262.51826524734497, + "token_usage": { + "input_tokens": 727262, + "output_tokens": 6392, + "total_tokens": 733654, + "reasoning_tokens": 0 + }, + "turn_count": 34 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 400.0, + "avg_input_tokens": 672115.8, + "avg_output_tokens": 10920.2, + "avg_total_tokens": 683036.0, + "avg_turn_count": 27.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 211.33162832260132, + "agent_execution_time": 211.33162832260132, + "token_usage": { + "input_tokens": 507721, + "output_tokens": 2507, + "total_tokens": 510228, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 449.900093793869, + "agent_execution_time": 449.900093793869, + "token_usage": { + "input_tokens": 554243, + "output_tokens": 9017, + "total_tokens": 563260, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-3", + "success": false, + "execution_time": 294.91173124313354, + "agent_execution_time": 294.91173124313354, + "token_usage": { + "input_tokens": 1017031, + "output_tokens": 3163, + "total_tokens": 1020194, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 198.33850598335266, + "agent_execution_time": 198.33850598335266, + "token_usage": { + "input_tokens": 287339, + "output_tokens": 2489, + "total_tokens": 289828, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 288.62, + "avg_input_tokens": 591583.5, + "avg_output_tokens": 4294.0, + "avg_total_tokens": 595877.5, + "avg_turn_count": 20.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 105.77533602714539, + "agent_execution_time": 105.77533602714539, + "token_usage": { + "input_tokens": 404023, + "output_tokens": 10149, + "total_tokens": 414172, + "reasoning_tokens": 7596 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 95.72051048278809, + "agent_execution_time": 95.72051048278809, + "token_usage": { + "input_tokens": 464662, + "output_tokens": 8073, + "total_tokens": 472735, + "reasoning_tokens": 6327 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": false, + "execution_time": 27.975849151611328, + "agent_execution_time": 27.975849151611328, + "token_usage": { + "input_tokens": 43906, + "output_tokens": 2619, + "total_tokens": 46525, + "reasoning_tokens": 2352 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 132.7514307498932, + "agent_execution_time": 132.7514307498932, + "token_usage": { + "input_tokens": 385557, + "output_tokens": 6907, + "total_tokens": 392464, + "reasoning_tokens": 5496 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 90.56, + "avg_input_tokens": 324537.0, + "avg_output_tokens": 6937.0, + "avg_total_tokens": 331474.0, + "avg_turn_count": 18.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/papers__author_folders.json b/task_results/papers__author_folders.json index 638177be..cc90a1d4 100644 --- a/task_results/papers__author_folders.json +++ b/task_results/papers__author_folders.json @@ -2,6 +2,73 @@ "task": "papers__author_folders", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 516.0934507846832, + "agent_execution_time": 516.0934507846832, + "token_usage": { + "input_tokens": 22534, + "output_tokens": 760, + "total_tokens": 23294, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 44.14991760253906, + "agent_execution_time": 44.14991760253906, + "token_usage": { + "input_tokens": 3356, + "output_tokens": 947, + "total_tokens": 4303, + "reasoning_tokens": 914 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1413.7718470096588, + "agent_execution_time": 1413.7718470096588, + "token_usage": { + "input_tokens": 33525, + "output_tokens": 14308, + "total_tokens": 47833, + "reasoning_tokens": 10709 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 419.44924664497375, + "agent_execution_time": 419.44924664497375, + "token_usage": { + "input_tokens": 11335, + "output_tokens": 10591, + "total_tokens": 21926, + "reasoning_tokens": 10486 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 598.37, + "avg_input_tokens": 17687.5, + "avg_output_tokens": 6651.5, + "avg_total_tokens": 24339.0, + "avg_turn_count": 3.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 656.0237100124359, - "agent_execution_time": 656.0237100124359, - "token_usage": { - "input_tokens": 1366936, - "output_tokens": 11172, - "total_tokens": 1378108, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 656.02, - "avg_input_tokens": 1366936.0, - "avg_output_tokens": 11172.0, - "avg_total_tokens": 1378108.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 123.42005062103271, + "agent_execution_time": 123.42005062103271, + "token_usage": { + "input_tokens": 1058921, + "output_tokens": 4521, + "total_tokens": 1063442, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 225.0844805240631, + "agent_execution_time": 225.0844805240631, + "token_usage": { + "input_tokens": 2154109, + "output_tokens": 10482, + "total_tokens": 2164591, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 54.932896852493286, + "agent_execution_time": 54.932896852493286, + "token_usage": { + "input_tokens": 77098, + "output_tokens": 1142, + "total_tokens": 78240, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 34.484243869781494, + "agent_execution_time": 34.484243869781494, + "token_usage": { + "input_tokens": 27922, + "output_tokens": 963, + "total_tokens": 28885, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 109.48, + "avg_input_tokens": 829512.5, + "avg_output_tokens": 4277.0, + "avg_total_tokens": 833789.5, + "avg_turn_count": 10.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 191.16957926750183, + "agent_execution_time": 191.16957926750183, + "token_usage": { + "input_tokens": 554552, + "output_tokens": 2263, + "total_tokens": 556815, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 30.366920232772827, + "agent_execution_time": 30.366920232772827, + "token_usage": { + "input_tokens": 123173, + "output_tokens": 1430, + "total_tokens": 124603, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 191.16957926750183, + "agent_execution_time": 191.16957926750183, + "token_usage": { + "input_tokens": 554552, + "output_tokens": 2263, + "total_tokens": 556815, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 30.366920232772827, + "agent_execution_time": 30.366920232772827, + "token_usage": { + "input_tokens": 123173, + "output_tokens": 1430, + "total_tokens": 124603, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 110.77, + "avg_input_tokens": 338862.5, + "avg_output_tokens": 1846.5, + "avg_total_tokens": 340709.0, + "avg_turn_count": 8.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 187.72283601760864, + "agent_execution_time": 187.72283601760864, + "token_usage": { + "input_tokens": 65623, + "output_tokens": 3502, + "total_tokens": 69125, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": false, + "execution_time": 63.74360632896423, + "agent_execution_time": 63.74360632896423, + "token_usage": { + "input_tokens": 66822, + "output_tokens": 736, + "total_tokens": 67558, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 266.04863381385803, + "agent_execution_time": 266.04863381385803, + "token_usage": { + "input_tokens": 130506, + "output_tokens": 725, + "total_tokens": 131231, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 259.42858505249023, + "agent_execution_time": 259.42858505249023, + "token_usage": { + "input_tokens": 11334, + "output_tokens": 545, + "total_tokens": 11879, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 194.24, + "avg_input_tokens": 68571.2, + "avg_output_tokens": 1377.0, + "avg_total_tokens": 69948.2, + "avg_turn_count": 4.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 412.186758518219, + "agent_execution_time": 412.186758518219, + "token_usage": { + "input_tokens": 5831585, + "output_tokens": 37565, + "total_tokens": 5869150, + "reasoning_tokens": 6750 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 98.62545466423035, + "agent_execution_time": 98.62545466423035, + "token_usage": { + "input_tokens": 809332, + "output_tokens": 8520, + "total_tokens": 817852, + "reasoning_tokens": 7183 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 830.5420525074005, + "agent_execution_time": 830.5420525074005, + "token_usage": { + "input_tokens": 21332562, + "output_tokens": 10848, + "total_tokens": 21343410, + "reasoning_tokens": 7027 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 221.21445202827454, + "agent_execution_time": 221.21445202827454, + "token_usage": { + "input_tokens": 2120484, + "output_tokens": 9674, + "total_tokens": 2130158, + "reasoning_tokens": 5969 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 390.64, + "avg_input_tokens": 7523490.8, + "avg_output_tokens": 16651.8, + "avg_total_tokens": 7540142.5, + "avg_turn_count": 11.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/papers__find_math_paper.json b/task_results/papers__find_math_paper.json index 7d9c8fbb..d093ccf9 100644 --- a/task_results/papers__find_math_paper.json +++ b/task_results/papers__find_math_paper.json @@ -2,6 +2,73 @@ "task": "papers__find_math_paper", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 375.5328314304352, + "agent_execution_time": 375.5328314304352, + "token_usage": { + "input_tokens": 112242, + "output_tokens": 832, + "total_tokens": 113074, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1449.2769982814789, + "agent_execution_time": 1449.2769982814789, + "token_usage": { + "input_tokens": 10141, + "output_tokens": 11865, + "total_tokens": 22006, + "reasoning_tokens": 8399 + }, + "turn_count": 3 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1257.1339843273163, + "agent_execution_time": 1257.1339843273163, + "token_usage": { + "input_tokens": 10141, + "output_tokens": 10952, + "total_tokens": 21093, + "reasoning_tokens": 7487 + }, + "turn_count": 3 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1301.1785910129547, + "agent_execution_time": 1301.1785910129547, + "token_usage": { + "input_tokens": 10141, + "output_tokens": 14357, + "total_tokens": 24498, + "reasoning_tokens": 10890 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1095.78, + "avg_input_tokens": 35666.2, + "avg_output_tokens": 9501.5, + "avg_total_tokens": 45167.8, + "avg_turn_count": 4.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 140.44481205940247, - "agent_execution_time": 140.44481205940247, - "token_usage": { - "input_tokens": 421671, - "output_tokens": 1378, - "total_tokens": 423049, - "reasoning_tokens": null - }, - "turn_count": 6 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 140.44, - "avg_input_tokens": 421671.0, - "avg_output_tokens": 1378.0, - "avg_total_tokens": 423049.0, - "avg_turn_count": 6.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 72.36091184616089, + "agent_execution_time": 72.36091184616089, + "token_usage": { + "input_tokens": 725195, + "output_tokens": 1548, + "total_tokens": 726743, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 95.37920951843262, + "agent_execution_time": 95.37920951843262, + "token_usage": { + "input_tokens": 767025, + "output_tokens": 3305, + "total_tokens": 770330, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 126.08392524719238, + "agent_execution_time": 126.08392524719238, + "token_usage": { + "input_tokens": 1237899, + "output_tokens": 3179, + "total_tokens": 1241078, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 90.26646852493286, + "agent_execution_time": 90.26646852493286, + "token_usage": { + "input_tokens": 741780, + "output_tokens": 2902, + "total_tokens": 744682, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 96.02, + "avg_input_tokens": 867974.8, + "avg_output_tokens": 2733.5, + "avg_total_tokens": 870708.2, + "avg_turn_count": 13.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 3298.3711700439453, + "agent_execution_time": 3298.3711700439453, + "token_usage": { + "input_tokens": 11920466, + "output_tokens": 57541, + "total_tokens": 11978007, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-2", + "success": false, + "execution_time": 2546.2523653507233, + "agent_execution_time": 2546.2523653507233, + "token_usage": { + "input_tokens": 11717689, + "output_tokens": 26675, + "total_tokens": 11744364, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-3", + "success": false, + "execution_time": 3298.3711700439453, + "agent_execution_time": 3298.3711700439453, + "token_usage": { + "input_tokens": 11920466, + "output_tokens": 57541, + "total_tokens": 11978007, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-4", + "success": false, + "execution_time": 2546.2523653507233, + "agent_execution_time": 2546.2523653507233, + "token_usage": { + "input_tokens": 11717689, + "output_tokens": 26675, + "total_tokens": 11744364, + "reasoning_tokens": 0 + }, + "turn_count": 100 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 2922.31, + "avg_input_tokens": 11819077.5, + "avg_output_tokens": 42108.0, + "avg_total_tokens": 11861185.5, + "avg_turn_count": 100.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 160.2336859703064, + "agent_execution_time": 160.2336859703064, + "token_usage": { + "input_tokens": 516390, + "output_tokens": 1067, + "total_tokens": 517457, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 194.11977434158325, + "agent_execution_time": 194.11977434158325, + "token_usage": { + "input_tokens": 105742, + "output_tokens": 680, + "total_tokens": 106422, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 398.12489771842957, + "agent_execution_time": 398.12489771842957, + "token_usage": { + "input_tokens": 387907, + "output_tokens": 834, + "total_tokens": 388741, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 229.76173663139343, + "agent_execution_time": 229.76173663139343, + "token_usage": { + "input_tokens": 10140, + "output_tokens": 450, + "total_tokens": 10590, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 245.56, + "avg_input_tokens": 255044.8, + "avg_output_tokens": 757.8, + "avg_total_tokens": 255802.5, + "avg_turn_count": 6.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 29.68813705444336, + "agent_execution_time": 29.68813705444336, + "token_usage": { + "input_tokens": 106082, + "output_tokens": 2087, + "total_tokens": 108169, + "reasoning_tokens": 1586 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": true, + "execution_time": 25.301192045211792, + "agent_execution_time": 25.301192045211792, + "token_usage": { + "input_tokens": 93055, + "output_tokens": 2903, + "total_tokens": 95958, + "reasoning_tokens": 2471 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": true, + "execution_time": 206.4177656173706, + "agent_execution_time": 206.4177656173706, + "token_usage": { + "input_tokens": 91771, + "output_tokens": 5099, + "total_tokens": 96870, + "reasoning_tokens": 4184 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": true, + "execution_time": 22.769134283065796, + "agent_execution_time": 22.769134283065796, + "token_usage": { + "input_tokens": 96312, + "output_tokens": 3170, + "total_tokens": 99482, + "reasoning_tokens": 2738 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 71.04, + "avg_input_tokens": 96805.0, + "avg_output_tokens": 3314.8, + "avg_total_tokens": 100119.8, + "avg_turn_count": 8.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/papers__organize_legacy_papers.json b/task_results/papers__organize_legacy_papers.json index acc3c707..a7afb29c 100644 --- a/task_results/papers__organize_legacy_papers.json +++ b/task_results/papers__organize_legacy_papers.json @@ -2,6 +2,73 @@ "task": "papers__organize_legacy_papers", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 540.0204980373383, + "agent_execution_time": 540.0204980373383, + "token_usage": { + "input_tokens": 39292, + "output_tokens": 1057, + "total_tokens": 40349, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 718.3258099555969, + "agent_execution_time": 718.3258099555969, + "token_usage": { + "input_tokens": 11953, + "output_tokens": 4826, + "total_tokens": 16779, + "reasoning_tokens": 4256 + }, + "turn_count": 3 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1577.7872943878174, + "agent_execution_time": 1577.7872943878174, + "token_usage": { + "input_tokens": 90002, + "output_tokens": 17456, + "total_tokens": 107458, + "reasoning_tokens": 14790 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 315.5271635055542, + "agent_execution_time": 315.5271635055542, + "token_usage": { + "input_tokens": 11952, + "output_tokens": 4239, + "total_tokens": 16191, + "reasoning_tokens": 3692 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 787.92, + "avg_input_tokens": 38299.8, + "avg_output_tokens": 6894.5, + "avg_total_tokens": 45194.2, + "avg_turn_count": 5.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 140.9607548713684, - "agent_execution_time": 140.9607548713684, - "token_usage": { - "input_tokens": 40663, - "output_tokens": 1636, - "total_tokens": 42299, - "reasoning_tokens": null - }, - "turn_count": 9 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 140.96, - "avg_input_tokens": 40663.0, - "avg_output_tokens": 1636.0, - "avg_total_tokens": 42299.0, - "avg_turn_count": 9.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 33.34410619735718, + "agent_execution_time": 33.34410619735718, + "token_usage": { + "input_tokens": 17860, + "output_tokens": 922, + "total_tokens": 18782, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": false, + "execution_time": 41.60207962989807, + "agent_execution_time": 41.60207962989807, + "token_usage": { + "input_tokens": 21516, + "output_tokens": 919, + "total_tokens": 22435, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 40.0706422328949, + "agent_execution_time": 40.0706422328949, + "token_usage": { + "input_tokens": 17903, + "output_tokens": 974, + "total_tokens": 18877, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 25.571825742721558, + "agent_execution_time": 25.571825742721558, + "token_usage": { + "input_tokens": 10866, + "output_tokens": 852, + "total_tokens": 11718, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 35.15, + "avg_input_tokens": 17036.2, + "avg_output_tokens": 916.8, + "avg_total_tokens": 17953.0, + "avg_turn_count": 3.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 236.41677069664001, + "agent_execution_time": 236.41677069664001, + "token_usage": { + "input_tokens": 955019, + "output_tokens": 6264, + "total_tokens": 961283, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 227.79384636878967, + "agent_execution_time": 227.79384636878967, + "token_usage": { + "input_tokens": 2429751, + "output_tokens": 5153, + "total_tokens": 2434904, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-3", + "success": false, + "execution_time": 236.41677069664001, + "agent_execution_time": 236.41677069664001, + "token_usage": { + "input_tokens": 955019, + "output_tokens": 6264, + "total_tokens": 961283, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": false, + "execution_time": 227.79384636878967, + "agent_execution_time": 227.79384636878967, + "token_usage": { + "input_tokens": 2429751, + "output_tokens": 5153, + "total_tokens": 2434904, + "reasoning_tokens": 0 + }, + "turn_count": 34 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 232.11, + "avg_input_tokens": 1692385.0, + "avg_output_tokens": 5708.5, + "avg_total_tokens": 1698093.5, + "avg_turn_count": 30.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 150.26462125778198, + "agent_execution_time": 150.26462125778198, + "token_usage": { + "input_tokens": 523809, + "output_tokens": 1220, + "total_tokens": 525029, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 79.78834533691406, + "agent_execution_time": 79.78834533691406, + "token_usage": { + "input_tokens": 85858, + "output_tokens": 830, + "total_tokens": 86688, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 6.471507787704468, + "agent_execution_time": 6.471507787704468, + "token_usage": { + "input_tokens": 3559, + "output_tokens": 33, + "total_tokens": 3592, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 340.7232069969177, + "agent_execution_time": 340.7232069969177, + "token_usage": { + "input_tokens": 11952, + "output_tokens": 555, + "total_tokens": 12507, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 144.31, + "avg_input_tokens": 156294.5, + "avg_output_tokens": 659.5, + "avg_total_tokens": 156954.0, + "avg_turn_count": 4.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 737.5350813865662, + "agent_execution_time": 737.5350813865662, + "token_usage": { + "input_tokens": 1651267, + "output_tokens": 7636, + "total_tokens": 1658903, + "reasoning_tokens": 3846 + }, + "turn_count": 35 + }, + { + "run": "run-2", + "success": false, + "execution_time": 68.53865361213684, + "agent_execution_time": 68.53865361213684, + "token_usage": { + "input_tokens": 1402969, + "output_tokens": 6299, + "total_tokens": 1409268, + "reasoning_tokens": 3152 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 99.52957820892334, + "agent_execution_time": 99.52957820892334, + "token_usage": { + "input_tokens": 1867993, + "output_tokens": 9540, + "total_tokens": 1877533, + "reasoning_tokens": 5838 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 235.27552676200867, + "agent_execution_time": 235.27552676200867, + "token_usage": { + "input_tokens": 7632065, + "output_tokens": 11999, + "total_tokens": 7644064, + "reasoning_tokens": 7876 + }, + "turn_count": 57 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 285.22, + "avg_input_tokens": 3138573.5, + "avg_output_tokens": 8868.5, + "avg_total_tokens": 3147442.0, + "avg_turn_count": 31.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/python_roadmap__expert_level_lessons.json b/task_results/python_roadmap__expert_level_lessons.json index ea07c5dc..f534d6f1 100644 --- a/task_results/python_roadmap__expert_level_lessons.json +++ b/task_results/python_roadmap__expert_level_lessons.json @@ -2,6 +2,73 @@ "task": "python_roadmap__expert_level_lessons", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 237.7851574420929, + "agent_execution_time": 237.7851574420929, + "token_usage": { + "input_tokens": 591789, + "output_tokens": 2760, + "total_tokens": 594549, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 2144.2917325496674, + "agent_execution_time": 2144.2917325496674, + "token_usage": { + "input_tokens": 352022, + "output_tokens": 33904, + "total_tokens": 385926, + "reasoning_tokens": 31153 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 97.71788716316223, + "agent_execution_time": 97.71788716316223, + "token_usage": { + "input_tokens": 154468, + "output_tokens": 879, + "total_tokens": 155347, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1801.4580142498016, + "agent_execution_time": 1801.4580142498016, + "token_usage": { + "input_tokens": 201954, + "output_tokens": 39304, + "total_tokens": 241258, + "reasoning_tokens": 36610 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1070.31, + "avg_input_tokens": 325058.2, + "avg_output_tokens": 19211.8, + "avg_total_tokens": 344270.0, + "avg_turn_count": 11.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 386.3332588672638, - "agent_execution_time": 386.3332588672638, - "token_usage": { - "input_tokens": 605665, - "output_tokens": 4641, - "total_tokens": 610306, - "reasoning_tokens": null - }, - "turn_count": 20 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 386.33, - "avg_input_tokens": 605665.0, - "avg_output_tokens": 4641.0, - "avg_total_tokens": 610306.0, - "avg_turn_count": 20.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 313.82976055145264, + "agent_execution_time": 313.82976055145264, + "token_usage": { + "input_tokens": 1732854, + "output_tokens": 10110, + "total_tokens": 1742964, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-2", + "success": false, + "execution_time": 182.64713835716248, + "agent_execution_time": 182.64713835716248, + "token_usage": { + "input_tokens": 910520, + "output_tokens": 6510, + "total_tokens": 917030, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": true, + "execution_time": 183.46198868751526, + "agent_execution_time": 183.46198868751526, + "token_usage": { + "input_tokens": 1153274, + "output_tokens": 6006, + "total_tokens": 1159280, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 334.6447241306305, + "agent_execution_time": 334.6447241306305, + "token_usage": { + "input_tokens": 1469449, + "output_tokens": 6879, + "total_tokens": 1476328, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 253.65, + "avg_input_tokens": 1316524.2, + "avg_output_tokens": 7376.2, + "avg_total_tokens": 1323900.5, + "avg_turn_count": 23.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 260.8379805088043, + "agent_execution_time": 260.8379805088043, + "token_usage": { + "input_tokens": 1511897, + "output_tokens": 4258, + "total_tokens": 1516155, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-2", + "success": true, + "execution_time": 205.4099669456482, + "agent_execution_time": 205.4099669456482, + "token_usage": { + "input_tokens": 932194, + "output_tokens": 3868, + "total_tokens": 936062, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-3", + "success": true, + "execution_time": 599.354977607727, + "agent_execution_time": 599.354977607727, + "token_usage": { + "input_tokens": 779376, + "output_tokens": 6302, + "total_tokens": 785678, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": true, + "execution_time": 263.7162415981293, + "agent_execution_time": 263.7162415981293, + "token_usage": { + "input_tokens": 1523435, + "output_tokens": 6169, + "total_tokens": 1529604, + "reasoning_tokens": 0 + }, + "turn_count": 30 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 332.33, + "avg_input_tokens": 1186725.5, + "avg_output_tokens": 5149.2, + "avg_total_tokens": 1191874.8, + "avg_turn_count": 29.25, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 291.72464203834534, + "agent_execution_time": 291.72464203834534, + "token_usage": { + "input_tokens": 1057888, + "output_tokens": 3011, + "total_tokens": 1060899, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 285.7454922199249, + "agent_execution_time": 285.7454922199249, + "token_usage": { + "input_tokens": 703963, + "output_tokens": 3664, + "total_tokens": 707627, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 597.889726638794, + "agent_execution_time": 597.889726638794, + "token_usage": { + "input_tokens": 3241155, + "output_tokens": 5440, + "total_tokens": 3246595, + "reasoning_tokens": 0 + }, + "turn_count": 43 + }, + { + "run": "run-4", + "success": false, + "execution_time": 327.34903502464294, + "agent_execution_time": 327.34903502464294, + "token_usage": { + "input_tokens": 1442443, + "output_tokens": 3443, + "total_tokens": 1445886, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 375.68, + "avg_input_tokens": 1611362.2, + "avg_output_tokens": 3889.5, + "avg_total_tokens": 1615251.8, + "avg_turn_count": 26.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 92.83392596244812, + "agent_execution_time": 92.83392596244812, + "token_usage": { + "input_tokens": 392706, + "output_tokens": 6863, + "total_tokens": 399569, + "reasoning_tokens": 4876 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 94.1509051322937, + "agent_execution_time": 94.1509051322937, + "token_usage": { + "input_tokens": 361773, + "output_tokens": 7281, + "total_tokens": 369054, + "reasoning_tokens": 5634 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": false, + "execution_time": 104.47433710098267, + "agent_execution_time": 104.47433710098267, + "token_usage": { + "input_tokens": 174030, + "output_tokens": 10573, + "total_tokens": 184603, + "reasoning_tokens": 8293 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 118.02387833595276, + "agent_execution_time": 118.02387833595276, + "token_usage": { + "input_tokens": 553560, + "output_tokens": 5849, + "total_tokens": 559409, + "reasoning_tokens": 4391 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 102.37, + "avg_input_tokens": 370517.2, + "avg_output_tokens": 7641.5, + "avg_total_tokens": 378158.8, + "avg_turn_count": 21.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/python_roadmap__learning_metrics_dashboard.json b/task_results/python_roadmap__learning_metrics_dashboard.json index b1057ed2..fea1fe1e 100644 --- a/task_results/python_roadmap__learning_metrics_dashboard.json +++ b/task_results/python_roadmap__learning_metrics_dashboard.json @@ -2,6 +2,73 @@ "task": "python_roadmap__learning_metrics_dashboard", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 167.33575439453125, + "agent_execution_time": 167.33575439453125, + "token_usage": { + "input_tokens": 355710, + "output_tokens": 2000, + "total_tokens": 357710, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1292.4404141902924, + "agent_execution_time": 1292.4404141902924, + "token_usage": { + "input_tokens": 71243, + "output_tokens": 32256, + "total_tokens": 103499, + "reasoning_tokens": 31861 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 155.6458375453949, + "agent_execution_time": 155.6458375453949, + "token_usage": { + "input_tokens": 368623, + "output_tokens": 1947, + "total_tokens": 370570, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 2841.3382205963135, + "agent_execution_time": 2841.3382205963135, + "token_usage": { + "input_tokens": 341306, + "output_tokens": 66012, + "total_tokens": 407318, + "reasoning_tokens": 60344 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1114.19, + "avg_input_tokens": 284220.5, + "avg_output_tokens": 25553.8, + "avg_total_tokens": 309774.2, + "avg_turn_count": 9.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 272.8607907295227, - "agent_execution_time": 272.8607907295227, - "token_usage": { - "input_tokens": 364570, - "output_tokens": 4207, - "total_tokens": 368777, - "reasoning_tokens": null - }, - "turn_count": 11 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 272.86, - "avg_input_tokens": 364570.0, - "avg_output_tokens": 4207.0, - "avg_total_tokens": 368777.0, - "avg_turn_count": 11.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 165.26004815101624, + "agent_execution_time": 165.26004815101624, + "token_usage": { + "input_tokens": 996104, + "output_tokens": 4752, + "total_tokens": 1000856, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 125.44312357902527, + "agent_execution_time": 125.44312357902527, + "token_usage": { + "input_tokens": 702217, + "output_tokens": 3752, + "total_tokens": 705969, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": false, + "execution_time": 176.05701327323914, + "agent_execution_time": 176.05701327323914, + "token_usage": { + "input_tokens": 1190124, + "output_tokens": 5361, + "total_tokens": 1195485, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 301.2488284111023, + "agent_execution_time": 301.2488284111023, + "token_usage": { + "input_tokens": 890721, + "output_tokens": 7647, + "total_tokens": 898368, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 192.0, + "avg_input_tokens": 944791.5, + "avg_output_tokens": 5378.0, + "avg_total_tokens": 950169.5, + "avg_turn_count": 17.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 131.44664645195007, + "agent_execution_time": 131.44664645195007, + "token_usage": { + "input_tokens": 292048, + "output_tokens": 2238, + "total_tokens": 294286, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 163.04621577262878, + "agent_execution_time": 163.04621577262878, + "token_usage": { + "input_tokens": 935700, + "output_tokens": 3513, + "total_tokens": 939213, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-3", + "success": false, + "execution_time": 227.32916402816772, + "agent_execution_time": 227.32916402816772, + "token_usage": { + "input_tokens": 358247, + "output_tokens": 3363, + "total_tokens": 361610, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": false, + "execution_time": 135.63160467147827, + "agent_execution_time": 135.63160467147827, + "token_usage": { + "input_tokens": 281897, + "output_tokens": 3981, + "total_tokens": 285878, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 164.36, + "avg_input_tokens": 466973.0, + "avg_output_tokens": 3273.8, + "avg_total_tokens": 470246.8, + "avg_turn_count": 16.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 305.62830877304077, + "agent_execution_time": 305.62830877304077, + "token_usage": { + "input_tokens": 1164054, + "output_tokens": 3044, + "total_tokens": 1167098, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": false, + "execution_time": 199.27596855163574, + "agent_execution_time": 199.27596855163574, + "token_usage": { + "input_tokens": 411218, + "output_tokens": 2996, + "total_tokens": 414214, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 395.6280062198639, + "agent_execution_time": 395.6280062198639, + "token_usage": { + "input_tokens": 1793209, + "output_tokens": 4069, + "total_tokens": 1797278, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": false, + "execution_time": 215.58000874519348, + "agent_execution_time": 215.58000874519348, + "token_usage": { + "input_tokens": 310733, + "output_tokens": 3078, + "total_tokens": 313811, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 279.03, + "avg_input_tokens": 919803.5, + "avg_output_tokens": 3296.8, + "avg_total_tokens": 923100.2, + "avg_turn_count": 18.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 546.8645763397217, + "agent_execution_time": 546.8645763397217, + "token_usage": { + "input_tokens": 385843, + "output_tokens": 75802, + "total_tokens": 461645, + "reasoning_tokens": 3986 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 71.43365120887756, + "agent_execution_time": 71.43365120887756, + "token_usage": { + "input_tokens": 678601, + "output_tokens": 6206, + "total_tokens": 684807, + "reasoning_tokens": 4491 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 74.8179440498352, + "agent_execution_time": 74.8179440498352, + "token_usage": { + "input_tokens": 389052, + "output_tokens": 6685, + "total_tokens": 395737, + "reasoning_tokens": 5351 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 135.74409294128418, + "agent_execution_time": 135.74409294128418, + "token_usage": { + "input_tokens": 1456294, + "output_tokens": 9547, + "total_tokens": 1465841, + "reasoning_tokens": 6803 + }, + "turn_count": 32 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 207.22, + "avg_input_tokens": 727447.5, + "avg_output_tokens": 24560.0, + "avg_total_tokens": 752007.5, + "avg_turn_count": 19.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/reddit__ai_data_analyst.json b/task_results/reddit__ai_data_analyst.json index 6bb149b8..0d5c792a 100644 --- a/task_results/reddit__ai_data_analyst.json +++ b/task_results/reddit__ai_data_analyst.json @@ -2,6 +2,73 @@ "task": "reddit__ai_data_analyst", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 227.35113620758057, + "agent_execution_time": 227.35113620758057, + "token_usage": { + "input_tokens": 706160, + "output_tokens": 1251, + "total_tokens": 707411, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 265.55537009239197, + "agent_execution_time": 265.55537009239197, + "token_usage": { + "input_tokens": 759976, + "output_tokens": 1161, + "total_tokens": 761137, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1014.8283731937408, + "agent_execution_time": 1014.8283731937408, + "token_usage": { + "input_tokens": 468098, + "output_tokens": 14526, + "total_tokens": 482624, + "reasoning_tokens": 13823 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1022.283301115036, + "agent_execution_time": 1022.283301115036, + "token_usage": { + "input_tokens": 338388, + "output_tokens": 11658, + "total_tokens": 350046, + "reasoning_tokens": 10916 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 632.5, + "avg_input_tokens": 568155.5, + "avg_output_tokens": 7149.0, + "avg_total_tokens": 575304.5, + "avg_turn_count": 14.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 393.66253876686096, - "agent_execution_time": 393.66253876686096, - "token_usage": { - "input_tokens": 1307145, - "output_tokens": 2321, - "total_tokens": 1309466, - "reasoning_tokens": null - }, - "turn_count": 20 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 393.66, - "avg_input_tokens": 1307145.0, - "avg_output_tokens": 2321.0, - "avg_total_tokens": 1309466.0, - "avg_turn_count": 20.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 122.39360022544861, + "agent_execution_time": 122.39360022544861, + "token_usage": { + "input_tokens": 810885, + "output_tokens": 2543, + "total_tokens": 813428, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 175.28705477714539, + "agent_execution_time": 175.28705477714539, + "token_usage": { + "input_tokens": 1430998, + "output_tokens": 3082, + "total_tokens": 1434080, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 122.74756741523743, + "agent_execution_time": 122.74756741523743, + "token_usage": { + "input_tokens": 739859, + "output_tokens": 2124, + "total_tokens": 741983, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 155.65950226783752, + "agent_execution_time": 155.65950226783752, + "token_usage": { + "input_tokens": 1219293, + "output_tokens": 2811, + "total_tokens": 1222104, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 144.02, + "avg_input_tokens": 1050258.8, + "avg_output_tokens": 2640.0, + "avg_total_tokens": 1052898.8, + "avg_turn_count": 16.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 210.279381275177, + "agent_execution_time": 210.279381275177, + "token_usage": { + "input_tokens": 712821, + "output_tokens": 3192, + "total_tokens": 716013, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 300.31256318092346, + "agent_execution_time": 300.31256318092346, + "token_usage": { + "input_tokens": 574886, + "output_tokens": 3217, + "total_tokens": 578103, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": false, + "execution_time": 194.1217384338379, + "agent_execution_time": 194.1217384338379, + "token_usage": { + "input_tokens": 855805, + "output_tokens": 3326, + "total_tokens": 859131, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 224.93473482131958, + "agent_execution_time": 224.93473482131958, + "token_usage": { + "input_tokens": 1120308, + "output_tokens": 4000, + "total_tokens": 1124308, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 232.41, + "avg_input_tokens": 815955.0, + "avg_output_tokens": 3433.8, + "avg_total_tokens": 819388.8, + "avg_turn_count": 17.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 228.44413709640503, + "agent_execution_time": 228.44413709640503, + "token_usage": { + "input_tokens": 705107, + "output_tokens": 1087, + "total_tokens": 706194, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 203.9344620704651, + "agent_execution_time": 203.9344620704651, + "token_usage": { + "input_tokens": 501001, + "output_tokens": 965, + "total_tokens": 501966, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 36.51499533653259, + "agent_execution_time": 36.51499533653259, + "token_usage": { + "input_tokens": 19569, + "output_tokens": 268, + "total_tokens": 19837, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 564.9863498210907, + "agent_execution_time": 564.9863498210907, + "token_usage": { + "input_tokens": 674826, + "output_tokens": 1256, + "total_tokens": 676082, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 258.47, + "avg_input_tokens": 475125.8, + "avg_output_tokens": 894.0, + "avg_total_tokens": 476019.8, + "avg_turn_count": 13.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 108.78989100456238, + "agent_execution_time": 108.78989100456238, + "token_usage": { + "input_tokens": 1372601, + "output_tokens": 4798, + "total_tokens": 1377399, + "reasoning_tokens": 3510 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 128.2094602584839, + "agent_execution_time": 128.2094602584839, + "token_usage": { + "input_tokens": 1848277, + "output_tokens": 7176, + "total_tokens": 1855453, + "reasoning_tokens": 6042 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 91.6964066028595, + "agent_execution_time": 91.6964066028595, + "token_usage": { + "input_tokens": 1634327, + "output_tokens": 4762, + "total_tokens": 1639089, + "reasoning_tokens": 3781 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 113.4655659198761, + "agent_execution_time": 113.4655659198761, + "token_usage": { + "input_tokens": 1975478, + "output_tokens": 3899, + "total_tokens": 1979377, + "reasoning_tokens": 3028 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 110.54, + "avg_input_tokens": 1707670.8, + "avg_output_tokens": 5158.8, + "avg_total_tokens": 1712829.5, + "avg_turn_count": 19.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/reddit__budget_europe_travel.json b/task_results/reddit__budget_europe_travel.json index ed75cde9..fdb6c283 100644 --- a/task_results/reddit__budget_europe_travel.json +++ b/task_results/reddit__budget_europe_travel.json @@ -2,6 +2,73 @@ "task": "reddit__budget_europe_travel", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 318.70132970809937, + "agent_execution_time": 318.70132970809937, + "token_usage": { + "input_tokens": 777622, + "output_tokens": 2497, + "total_tokens": 780119, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-2", + "success": true, + "execution_time": 264.9030439853668, + "agent_execution_time": 264.9030439853668, + "token_usage": { + "input_tokens": 705056, + "output_tokens": 2418, + "total_tokens": 707474, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-3", + "success": true, + "execution_time": 1294.1408152580261, + "agent_execution_time": 1294.1408152580261, + "token_usage": { + "input_tokens": 693305, + "output_tokens": 19917, + "total_tokens": 713222, + "reasoning_tokens": 18071 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1498.355031967163, + "agent_execution_time": 1498.355031967163, + "token_usage": { + "input_tokens": 724980, + "output_tokens": 17927, + "total_tokens": 742907, + "reasoning_tokens": 15935 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 844.03, + "avg_input_tokens": 725240.8, + "avg_output_tokens": 10689.8, + "avg_total_tokens": 735930.5, + "avg_turn_count": 25.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 534.0608282089233, - "agent_execution_time": 534.0608282089233, - "token_usage": { - "input_tokens": 953450, - "output_tokens": 5090, - "total_tokens": 958540, - "reasoning_tokens": null - }, - "turn_count": 32 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 534.06, - "avg_input_tokens": 953450.0, - "avg_output_tokens": 5090.0, - "avg_total_tokens": 958540.0, - "avg_turn_count": 32.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 196.1325080394745, + "agent_execution_time": 196.1325080394745, + "token_usage": { + "input_tokens": 1064397, + "output_tokens": 4494, + "total_tokens": 1068891, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + { + "run": "run-2", + "success": true, + "execution_time": 205.26203393936157, + "agent_execution_time": 205.26203393936157, + "token_usage": { + "input_tokens": 888870, + "output_tokens": 4107, + "total_tokens": 892977, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-3", + "success": true, + "execution_time": 169.2699544429779, + "agent_execution_time": 169.2699544429779, + "token_usage": { + "input_tokens": 811080, + "output_tokens": 4110, + "total_tokens": 815190, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": false, + "execution_time": 236.92507910728455, + "agent_execution_time": 236.92507910728455, + "token_usage": { + "input_tokens": 1136974, + "output_tokens": 5089, + "total_tokens": 1142063, + "reasoning_tokens": 0 + }, + "turn_count": 36 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 201.9, + "avg_input_tokens": 975330.2, + "avg_output_tokens": 4450.0, + "avg_total_tokens": 979780.2, + "avg_turn_count": 32.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 266.541437625885, + "agent_execution_time": 266.541437625885, + "token_usage": { + "input_tokens": 646523, + "output_tokens": 4384, + "total_tokens": 650907, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": true, + "execution_time": 571.070365190506, + "agent_execution_time": 571.070365190506, + "token_usage": { + "input_tokens": 857087, + "output_tokens": 6130, + "total_tokens": 863217, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-3", + "success": true, + "execution_time": 178.58940267562866, + "agent_execution_time": 178.58940267562866, + "token_usage": { + "input_tokens": 736556, + "output_tokens": 4605, + "total_tokens": 741161, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": true, + "execution_time": 251.43928456306458, + "agent_execution_time": 251.43928456306458, + "token_usage": { + "input_tokens": 872092, + "output_tokens": 5732, + "total_tokens": 877824, + "reasoning_tokens": 0 + }, + "turn_count": 28 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 316.91, + "avg_input_tokens": 778064.5, + "avg_output_tokens": 5212.8, + "avg_total_tokens": 783277.2, + "avg_turn_count": 26.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 277.6534972190857, + "agent_execution_time": 277.6534972190857, + "token_usage": { + "input_tokens": 818825, + "output_tokens": 2398, + "total_tokens": 821223, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-2", + "success": true, + "execution_time": 271.2076871395111, + "agent_execution_time": 271.2076871395111, + "token_usage": { + "input_tokens": 742915, + "output_tokens": 2206, + "total_tokens": 745121, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-3", + "success": true, + "execution_time": 558.0852010250092, + "agent_execution_time": 558.0852010250092, + "token_usage": { + "input_tokens": 857847, + "output_tokens": 2212, + "total_tokens": 860059, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-4", + "success": true, + "execution_time": 847.8150887489319, + "agent_execution_time": 847.8150887489319, + "token_usage": { + "input_tokens": 1396152, + "output_tokens": 2480, + "total_tokens": 1398632, + "reasoning_tokens": 0 + }, + "turn_count": 39 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 488.69, + "avg_input_tokens": 953934.8, + "avg_output_tokens": 2324.0, + "avg_total_tokens": 956258.8, + "avg_turn_count": 31.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 68.495689868927, + "agent_execution_time": 68.495689868927, + "token_usage": { + "input_tokens": 560735, + "output_tokens": 3500, + "total_tokens": 564235, + "reasoning_tokens": 2246 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": true, + "execution_time": 75.1107017993927, + "agent_execution_time": 75.1107017993927, + "token_usage": { + "input_tokens": 492329, + "output_tokens": 3031, + "total_tokens": 495360, + "reasoning_tokens": 1769 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": true, + "execution_time": 74.9792993068695, + "agent_execution_time": 74.9792993068695, + "token_usage": { + "input_tokens": 505275, + "output_tokens": 3250, + "total_tokens": 508525, + "reasoning_tokens": 1764 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 72.11851787567139, + "agent_execution_time": 72.11851787567139, + "token_usage": { + "input_tokens": 494449, + "output_tokens": 3332, + "total_tokens": 497781, + "reasoning_tokens": 2074 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 72.68, + "avg_input_tokens": 513197.0, + "avg_output_tokens": 3278.2, + "avg_total_tokens": 516475.2, + "avg_turn_count": 20.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/reddit__buyitforlife_research.json b/task_results/reddit__buyitforlife_research.json index 30ddccd3..71ac7fc1 100644 --- a/task_results/reddit__buyitforlife_research.json +++ b/task_results/reddit__buyitforlife_research.json @@ -2,6 +2,73 @@ "task": "reddit__buyitforlife_research", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 485.3865087032318, + "agent_execution_time": 485.3865087032318, + "token_usage": { + "input_tokens": 233867, + "output_tokens": 800, + "total_tokens": 234667, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 146.7022669315338, + "agent_execution_time": 146.7022669315338, + "token_usage": { + "input_tokens": 234353, + "output_tokens": 855, + "total_tokens": 235208, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 187.0119686126709, + "agent_execution_time": 187.0119686126709, + "token_usage": { + "input_tokens": 33241, + "output_tokens": 3757, + "total_tokens": 36998, + "reasoning_tokens": 3457 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 696.3547940254211, + "agent_execution_time": 696.3547940254211, + "token_usage": { + "input_tokens": 126105, + "output_tokens": 7768, + "total_tokens": 133873, + "reasoning_tokens": 7290 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 378.86, + "avg_input_tokens": 156891.5, + "avg_output_tokens": 3295.0, + "avg_total_tokens": 160186.5, + "avg_turn_count": 8.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 417.5229127407074, - "agent_execution_time": 417.5229127407074, - "token_usage": { - "input_tokens": 1596799, - "output_tokens": 2703, - "total_tokens": 1599502, - "reasoning_tokens": null - }, - "turn_count": 17 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 417.52, - "avg_input_tokens": 1596799.0, - "avg_output_tokens": 2703.0, - "avg_total_tokens": 1599502.0, - "avg_turn_count": 17.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 157.67092394828796, + "agent_execution_time": 157.67092394828796, + "token_usage": { + "input_tokens": 1323550, + "output_tokens": 2816, + "total_tokens": 1326366, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 128.9041256904602, + "agent_execution_time": 128.9041256904602, + "token_usage": { + "input_tokens": 607159, + "output_tokens": 2170, + "total_tokens": 609329, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": true, + "execution_time": 134.75087237358093, + "agent_execution_time": 134.75087237358093, + "token_usage": { + "input_tokens": 1136386, + "output_tokens": 2832, + "total_tokens": 1139218, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 115.47299408912659, + "agent_execution_time": 115.47299408912659, + "token_usage": { + "input_tokens": 625726, + "output_tokens": 1954, + "total_tokens": 627680, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 134.2, + "avg_input_tokens": 923205.2, + "avg_output_tokens": 2443.0, + "avg_total_tokens": 925648.2, + "avg_turn_count": 15.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 163.68683624267578, + "agent_execution_time": 163.68683624267578, + "token_usage": { + "input_tokens": 481128, + "output_tokens": 1675, + "total_tokens": 482803, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 169.3498990535736, + "agent_execution_time": 169.3498990535736, + "token_usage": { + "input_tokens": 355013, + "output_tokens": 2337, + "total_tokens": 357350, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 119.77276492118835, + "agent_execution_time": 119.77276492118835, + "token_usage": { + "input_tokens": 399690, + "output_tokens": 2167, + "total_tokens": 401857, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 101.64448142051697, + "agent_execution_time": 101.64448142051697, + "token_usage": { + "input_tokens": 307110, + "output_tokens": 1842, + "total_tokens": 308952, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 138.61, + "avg_input_tokens": 385735.2, + "avg_output_tokens": 2005.2, + "avg_total_tokens": 387740.5, + "avg_turn_count": 11.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 119.06816720962524, + "agent_execution_time": 119.06816720962524, + "token_usage": { + "input_tokens": 233113, + "output_tokens": 727, + "total_tokens": 233840, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 120.34467816352844, + "agent_execution_time": 120.34467816352844, + "token_usage": { + "input_tokens": 233768, + "output_tokens": 723, + "total_tokens": 234491, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 392.5510013103485, + "agent_execution_time": 392.5510013103485, + "token_usage": { + "input_tokens": 391180, + "output_tokens": 711, + "total_tokens": 391891, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": false, + "execution_time": 566.3692433834076, + "agent_execution_time": 566.3692433834076, + "token_usage": { + "input_tokens": 868222, + "output_tokens": 1998, + "total_tokens": 870220, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 299.58, + "avg_input_tokens": 431570.8, + "avg_output_tokens": 1039.8, + "avg_total_tokens": 432610.5, + "avg_turn_count": 13.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 57.46671414375305, + "agent_execution_time": 57.46671414375305, + "token_usage": { + "input_tokens": 443162, + "output_tokens": 2287, + "total_tokens": 445449, + "reasoning_tokens": 1608 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 68.92222189903259, + "agent_execution_time": 68.92222189903259, + "token_usage": { + "input_tokens": 1048576, + "output_tokens": 3266, + "total_tokens": 1051842, + "reasoning_tokens": 2440 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 61.987855672836304, + "agent_execution_time": 61.987855672836304, + "token_usage": { + "input_tokens": 731634, + "output_tokens": 3252, + "total_tokens": 734886, + "reasoning_tokens": 2562 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": false, + "execution_time": 84.4476706981659, + "agent_execution_time": 84.4476706981659, + "token_usage": { + "input_tokens": 602674, + "output_tokens": 3535, + "total_tokens": 606209, + "reasoning_tokens": 2405 + }, + "turn_count": 16 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 68.21, + "avg_input_tokens": 706511.5, + "avg_output_tokens": 3085.0, + "avg_total_tokens": 709596.5, + "avg_turn_count": 13.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/reddit__llm_research_summary.json b/task_results/reddit__llm_research_summary.json index ba3f9d5d..7f4a5ef1 100644 --- a/task_results/reddit__llm_research_summary.json +++ b/task_results/reddit__llm_research_summary.json @@ -2,6 +2,73 @@ "task": "reddit__llm_research_summary", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 806.8626036643982, + "agent_execution_time": 806.8626036643982, + "token_usage": { + "input_tokens": 783458, + "output_tokens": 4273, + "total_tokens": 787731, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 272.061527967453, + "agent_execution_time": 272.061527967453, + "token_usage": { + "input_tokens": 583775, + "output_tokens": 1834, + "total_tokens": 585609, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 2122.9438638687134, + "agent_execution_time": 2122.9438638687134, + "token_usage": { + "input_tokens": 287862, + "output_tokens": 36631, + "total_tokens": 324493, + "reasoning_tokens": 30859 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1961.2656562328339, + "agent_execution_time": 1961.2656562328339, + "token_usage": { + "input_tokens": 485983, + "output_tokens": 47180, + "total_tokens": 533163, + "reasoning_tokens": 45435 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1290.78, + "avg_input_tokens": 535269.5, + "avg_output_tokens": 22479.5, + "avg_total_tokens": 557749.0, + "avg_turn_count": 13.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 403.86750411987305, - "agent_execution_time": 403.86750411987305, - "token_usage": { - "input_tokens": 984735, - "output_tokens": 3636, - "total_tokens": 988371, - "reasoning_tokens": null - }, - "turn_count": 20 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 403.87, - "avg_input_tokens": 984735.0, - "avg_output_tokens": 3636.0, - "avg_total_tokens": 988371.0, - "avg_turn_count": 20.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 96.99859285354614, + "agent_execution_time": 96.99859285354614, + "token_usage": { + "input_tokens": 473634, + "output_tokens": 2536, + "total_tokens": 476170, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 166.61272382736206, + "agent_execution_time": 166.61272382736206, + "token_usage": { + "input_tokens": 1492546, + "output_tokens": 4520, + "total_tokens": 1497066, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 186.2016956806183, + "agent_execution_time": 186.2016956806183, + "token_usage": { + "input_tokens": 1574490, + "output_tokens": 3844, + "total_tokens": 1578334, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 172.61223649978638, + "agent_execution_time": 172.61223649978638, + "token_usage": { + "input_tokens": 1512269, + "output_tokens": 4259, + "total_tokens": 1516528, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 155.61, + "avg_input_tokens": 1263234.8, + "avg_output_tokens": 3789.8, + "avg_total_tokens": 1267024.5, + "avg_turn_count": 17.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 245.22378993034363, + "agent_execution_time": 245.22378993034363, + "token_usage": { + "input_tokens": 515961, + "output_tokens": 4574, + "total_tokens": 520535, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 220.0705966949463, + "agent_execution_time": 220.0705966949463, + "token_usage": { + "input_tokens": 872219, + "output_tokens": 5022, + "total_tokens": 877241, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 178.38008904457092, + "agent_execution_time": 178.38008904457092, + "token_usage": { + "input_tokens": 747936, + "output_tokens": 4806, + "total_tokens": 752742, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 255.52719640731812, + "agent_execution_time": 255.52719640731812, + "token_usage": { + "input_tokens": 520757, + "output_tokens": 4694, + "total_tokens": 525451, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 224.8, + "avg_input_tokens": 664218.2, + "avg_output_tokens": 4774.0, + "avg_total_tokens": 668992.2, + "avg_turn_count": 16.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 236.75895142555237, + "agent_execution_time": 236.75895142555237, + "token_usage": { + "input_tokens": 726255, + "output_tokens": 1749, + "total_tokens": 728004, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 300.7061505317688, + "agent_execution_time": 300.7061505317688, + "token_usage": { + "input_tokens": 835872, + "output_tokens": 3337, + "total_tokens": 839209, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 12.010667562484741, + "agent_execution_time": 12.010667562484741, + "token_usage": { + "input_tokens": 4437, + "output_tokens": 176, + "total_tokens": 4613, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 842.587836265564, + "agent_execution_time": 842.587836265564, + "token_usage": { + "input_tokens": 971342, + "output_tokens": 2628, + "total_tokens": 973970, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 348.02, + "avg_input_tokens": 634476.5, + "avg_output_tokens": 1972.5, + "avg_total_tokens": 636449.0, + "avg_turn_count": 12.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 72.01405262947083, + "agent_execution_time": 72.01405262947083, + "token_usage": { + "input_tokens": 610637, + "output_tokens": 4634, + "total_tokens": 615271, + "reasoning_tokens": 3781 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 80.41897535324097, + "agent_execution_time": 80.41897535324097, + "token_usage": { + "input_tokens": 820520, + "output_tokens": 3751, + "total_tokens": 824271, + "reasoning_tokens": 2774 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 68.50351619720459, + "agent_execution_time": 68.50351619720459, + "token_usage": { + "input_tokens": 591644, + "output_tokens": 4397, + "total_tokens": 596041, + "reasoning_tokens": 3566 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 60.905073165893555, + "agent_execution_time": 60.905073165893555, + "token_usage": { + "input_tokens": 336646, + "output_tokens": 4254, + "total_tokens": 340900, + "reasoning_tokens": 3560 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 70.46, + "avg_input_tokens": 589861.8, + "avg_output_tokens": 4259.0, + "avg_total_tokens": 594120.8, + "avg_turn_count": 16.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/reddit__movie_reviewer_analysis.json b/task_results/reddit__movie_reviewer_analysis.json index 6172facc..f315c2f9 100644 --- a/task_results/reddit__movie_reviewer_analysis.json +++ b/task_results/reddit__movie_reviewer_analysis.json @@ -2,6 +2,73 @@ "task": "reddit__movie_reviewer_analysis", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 587.6891911029816, + "agent_execution_time": 587.6891911029816, + "token_usage": { + "input_tokens": 675092, + "output_tokens": 1879, + "total_tokens": 676971, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 276.3106234073639, + "agent_execution_time": 276.3106234073639, + "token_usage": { + "input_tokens": 554078, + "output_tokens": 2416, + "total_tokens": 556494, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 3022.8521876335144, + "agent_execution_time": 3022.8521876335144, + "token_usage": { + "input_tokens": 587727, + "output_tokens": 72481, + "total_tokens": 660208, + "reasoning_tokens": 71015 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1093.3647103309631, + "agent_execution_time": 1093.3647103309631, + "token_usage": { + "input_tokens": 177955, + "output_tokens": 28813, + "total_tokens": 206768, + "reasoning_tokens": 27934 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1245.05, + "avg_input_tokens": 498713.0, + "avg_output_tokens": 26397.2, + "avg_total_tokens": 525110.2, + "avg_turn_count": 12.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 357.2769422531128, - "agent_execution_time": 357.2769422531128, - "token_usage": { - "input_tokens": 930347, - "output_tokens": 3140, - "total_tokens": 933487, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 357.28, - "avg_input_tokens": 930347.0, - "avg_output_tokens": 3140.0, - "avg_total_tokens": 933487.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 153.9569227695465, + "agent_execution_time": 153.9569227695465, + "token_usage": { + "input_tokens": 753348, + "output_tokens": 4743, + "total_tokens": 758091, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 146.11267495155334, + "agent_execution_time": 146.11267495155334, + "token_usage": { + "input_tokens": 764902, + "output_tokens": 4362, + "total_tokens": 769264, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": false, + "execution_time": 219.72306609153748, + "agent_execution_time": 219.72306609153748, + "token_usage": { + "input_tokens": 1067686, + "output_tokens": 4512, + "total_tokens": 1072198, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 156.99191880226135, + "agent_execution_time": 156.99191880226135, + "token_usage": { + "input_tokens": 947009, + "output_tokens": 3929, + "total_tokens": 950938, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 169.2, + "avg_input_tokens": 883236.2, + "avg_output_tokens": 4386.5, + "avg_total_tokens": 887622.8, + "avg_turn_count": 16.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 996.4635088443756, + "agent_execution_time": 996.4635088443756, + "token_usage": { + "input_tokens": 922761, + "output_tokens": 6450, + "total_tokens": 929211, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 292.49811720848083, + "agent_execution_time": 292.49811720848083, + "token_usage": { + "input_tokens": 1075048, + "output_tokens": 4264, + "total_tokens": 1079312, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 349.60808777809143, + "agent_execution_time": 349.60808777809143, + "token_usage": { + "input_tokens": 1496450, + "output_tokens": 5599, + "total_tokens": 1502049, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 274.4761805534363, + "agent_execution_time": 274.4761805534363, + "token_usage": { + "input_tokens": 692459, + "output_tokens": 5781, + "total_tokens": 698240, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 478.26, + "avg_input_tokens": 1046679.5, + "avg_output_tokens": 5523.5, + "avg_total_tokens": 1052203.0, + "avg_turn_count": 16.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 275.7007510662079, + "agent_execution_time": 275.7007510662079, + "token_usage": { + "input_tokens": 674460, + "output_tokens": 2511, + "total_tokens": 676971, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 299.4474368095398, + "agent_execution_time": 299.4474368095398, + "token_usage": { + "input_tokens": 800781, + "output_tokens": 3649, + "total_tokens": 804430, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 662.4497663974762, + "agent_execution_time": 662.4497663974762, + "token_usage": { + "input_tokens": 692600, + "output_tokens": 3833, + "total_tokens": 696433, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 766.1712210178375, + "agent_execution_time": 766.1712210178375, + "token_usage": { + "input_tokens": 1037160, + "output_tokens": 3055, + "total_tokens": 1040215, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 500.94, + "avg_input_tokens": 801250.2, + "avg_output_tokens": 3262.0, + "avg_total_tokens": 804512.2, + "avg_turn_count": 15.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 104.28021621704102, + "agent_execution_time": 104.28021621704102, + "token_usage": { + "input_tokens": 787367, + "output_tokens": 8412, + "total_tokens": 795779, + "reasoning_tokens": 7504 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 69.53013396263123, + "agent_execution_time": 69.53013396263123, + "token_usage": { + "input_tokens": 576625, + "output_tokens": 5392, + "total_tokens": 582017, + "reasoning_tokens": 4554 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": false, + "execution_time": 63.78611612319946, + "agent_execution_time": 63.78611612319946, + "token_usage": { + "input_tokens": 624287, + "output_tokens": 6413, + "total_tokens": 630700, + "reasoning_tokens": 5578 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 79.39650273323059, + "agent_execution_time": 79.39650273323059, + "token_usage": { + "input_tokens": 871617, + "output_tokens": 5629, + "total_tokens": 877246, + "reasoning_tokens": 4663 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 79.25, + "avg_input_tokens": 714974.0, + "avg_output_tokens": 6461.5, + "avg_total_tokens": 721435.5, + "avg_turn_count": 15.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/reddit__nba_statistics_analysis.json b/task_results/reddit__nba_statistics_analysis.json index 33cc93d1..4961c5da 100644 --- a/task_results/reddit__nba_statistics_analysis.json +++ b/task_results/reddit__nba_statistics_analysis.json @@ -2,6 +2,73 @@ "task": "reddit__nba_statistics_analysis", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 605.2492165565491, + "agent_execution_time": 605.2492165565491, + "token_usage": { + "input_tokens": 934805, + "output_tokens": 2383, + "total_tokens": 937188, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 224.2045509815216, + "agent_execution_time": 224.2045509815216, + "token_usage": { + "input_tokens": 335957, + "output_tokens": 1736, + "total_tokens": 337693, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1158.9364974498749, + "agent_execution_time": 1158.9364974498749, + "token_usage": { + "input_tokens": 275592, + "output_tokens": 17246, + "total_tokens": 292838, + "reasoning_tokens": 16681 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1046.6779713630676, + "agent_execution_time": 1046.6779713630676, + "token_usage": { + "input_tokens": 240010, + "output_tokens": 26666, + "total_tokens": 266676, + "reasoning_tokens": 26054 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 758.77, + "avg_input_tokens": 446591.0, + "avg_output_tokens": 12007.8, + "avg_total_tokens": 458598.8, + "avg_turn_count": 11.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 377.93063974380493, - "agent_execution_time": 377.93063974380493, - "token_usage": { - "input_tokens": 1224076, - "output_tokens": 2161, - "total_tokens": 1226237, - "reasoning_tokens": null - }, - "turn_count": 19 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 377.93, - "avg_input_tokens": 1224076.0, - "avg_output_tokens": 2161.0, - "avg_total_tokens": 1226237.0, - "avg_turn_count": 19.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 184.2265748977661, + "agent_execution_time": 184.2265748977661, + "token_usage": { + "input_tokens": 1749983, + "output_tokens": 4226, + "total_tokens": 1754209, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 205.5121591091156, + "agent_execution_time": 205.5121591091156, + "token_usage": { + "input_tokens": 1649711, + "output_tokens": 4362, + "total_tokens": 1654073, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 262.53530740737915, + "agent_execution_time": 262.53530740737915, + "token_usage": { + "input_tokens": 1455001, + "output_tokens": 3098, + "total_tokens": 1458099, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 198.60794973373413, + "agent_execution_time": 198.60794973373413, + "token_usage": { + "input_tokens": 1853255, + "output_tokens": 3791, + "total_tokens": 1857046, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 212.72, + "avg_input_tokens": 1676987.5, + "avg_output_tokens": 3869.2, + "avg_total_tokens": 1680856.8, + "avg_turn_count": 20.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 288.4777319431305, + "agent_execution_time": 288.4777319431305, + "token_usage": { + "input_tokens": 1011449, + "output_tokens": 1882, + "total_tokens": 1013331, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 180.97325587272644, + "agent_execution_time": 180.97325587272644, + "token_usage": { + "input_tokens": 526313, + "output_tokens": 4054, + "total_tokens": 530367, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 212.10700798034668, + "agent_execution_time": 212.10700798034668, + "token_usage": { + "input_tokens": 1067637, + "output_tokens": 4827, + "total_tokens": 1072464, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 350.5820713043213, + "agent_execution_time": 350.5820713043213, + "token_usage": { + "input_tokens": 998422, + "output_tokens": 4478, + "total_tokens": 1002900, + "reasoning_tokens": 0 + }, + "turn_count": 16 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 258.04, + "avg_input_tokens": 900955.2, + "avg_output_tokens": 3810.2, + "avg_total_tokens": 904765.5, + "avg_turn_count": 15.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 281.4182059764862, + "agent_execution_time": 281.4182059764862, + "token_usage": { + "input_tokens": 1192604, + "output_tokens": 1815, + "total_tokens": 1194419, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 223.13858580589294, + "agent_execution_time": 223.13858580589294, + "token_usage": { + "input_tokens": 541073, + "output_tokens": 1488, + "total_tokens": 542561, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 319.2330069541931, + "agent_execution_time": 319.2330069541931, + "token_usage": { + "input_tokens": 852795, + "output_tokens": 2327, + "total_tokens": 855122, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 15.212673664093018, + "agent_execution_time": 15.212673664093018, + "token_usage": { + "input_tokens": 4488, + "output_tokens": 190, + "total_tokens": 4678, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 209.75, + "avg_input_tokens": 647740.0, + "avg_output_tokens": 1455.0, + "avg_total_tokens": 649195.0, + "avg_turn_count": 11.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 145.343403339386, + "agent_execution_time": 145.343403339386, + "token_usage": { + "input_tokens": 2894579, + "output_tokens": 8570, + "total_tokens": 2903149, + "reasoning_tokens": 6254 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 151.66982221603394, + "agent_execution_time": 151.66982221603394, + "token_usage": { + "input_tokens": 1697840, + "output_tokens": 10765, + "total_tokens": 1708605, + "reasoning_tokens": 8907 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 111.23613333702087, + "agent_execution_time": 111.23613333702087, + "token_usage": { + "input_tokens": 1073525, + "output_tokens": 6759, + "total_tokens": 1080284, + "reasoning_tokens": 5411 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 86.50196313858032, + "agent_execution_time": 86.50196313858032, + "token_usage": { + "input_tokens": 662372, + "output_tokens": 5806, + "total_tokens": 668178, + "reasoning_tokens": 4935 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 123.69, + "avg_input_tokens": 1582079.0, + "avg_output_tokens": 7975.0, + "avg_total_tokens": 1590054.0, + "avg_turn_count": 19.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/reddit__routine_tracker_forum.json b/task_results/reddit__routine_tracker_forum.json index 5e1801b6..bcb9be40 100644 --- a/task_results/reddit__routine_tracker_forum.json +++ b/task_results/reddit__routine_tracker_forum.json @@ -2,6 +2,73 @@ "task": "reddit__routine_tracker_forum", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 541.5869607925415, + "agent_execution_time": 541.5869607925415, + "token_usage": { + "input_tokens": 194865, + "output_tokens": 748, + "total_tokens": 195613, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 181.92701196670532, + "agent_execution_time": 181.92701196670532, + "token_usage": { + "input_tokens": 194847, + "output_tokens": 729, + "total_tokens": 195576, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 704.2251386642456, + "agent_execution_time": 704.2251386642456, + "token_usage": { + "input_tokens": 62899, + "output_tokens": 8453, + "total_tokens": 71352, + "reasoning_tokens": 7883 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 473.9642276763916, + "agent_execution_time": 473.9642276763916, + "token_usage": { + "input_tokens": 75383, + "output_tokens": 5653, + "total_tokens": 81036, + "reasoning_tokens": 5195 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 475.43, + "avg_input_tokens": 131998.5, + "avg_output_tokens": 3895.8, + "avg_total_tokens": 135894.2, + "avg_turn_count": 7.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 1.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 346.12610816955566, - "agent_execution_time": 346.12610816955566, - "token_usage": { - "input_tokens": 1220576, - "output_tokens": 2063, - "total_tokens": 1222639, - "reasoning_tokens": null - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 346.13, - "avg_input_tokens": 1220576.0, - "avg_output_tokens": 2063.0, - "avg_total_tokens": 1222639.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 127.6527099609375, + "agent_execution_time": 127.6527099609375, + "token_usage": { + "input_tokens": 1385788, + "output_tokens": 2353, + "total_tokens": 1388141, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": true, + "execution_time": 274.5775909423828, + "agent_execution_time": 274.5775909423828, + "token_usage": { + "input_tokens": 1424503, + "output_tokens": 2314, + "total_tokens": 1426817, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": true, + "execution_time": 239.11703038215637, + "agent_execution_time": 239.11703038215637, + "token_usage": { + "input_tokens": 1535007, + "output_tokens": 3000, + "total_tokens": 1538007, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": true, + "execution_time": 170.82811069488525, + "agent_execution_time": 170.82811069488525, + "token_usage": { + "input_tokens": 1610847, + "output_tokens": 2632, + "total_tokens": 1613479, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 203.04, + "avg_input_tokens": 1489036.2, + "avg_output_tokens": 2574.8, + "avg_total_tokens": 1491611.0, + "avg_turn_count": 17.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 176.59448647499084, + "agent_execution_time": 176.59448647499084, + "token_usage": { + "input_tokens": 207392, + "output_tokens": 1506, + "total_tokens": 208898, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 139.69563055038452, + "agent_execution_time": 139.69563055038452, + "token_usage": { + "input_tokens": 468396, + "output_tokens": 2132, + "total_tokens": 470528, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 248.80995178222656, + "agent_execution_time": 248.80995178222656, + "token_usage": { + "input_tokens": 1241486, + "output_tokens": 2527, + "total_tokens": 1244013, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 206.3200387954712, + "agent_execution_time": 206.3200387954712, + "token_usage": { + "input_tokens": 716039, + "output_tokens": 2094, + "total_tokens": 718133, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 192.86, + "avg_input_tokens": 658328.2, + "avg_output_tokens": 2064.8, + "avg_total_tokens": 660393.0, + "avg_turn_count": 12.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 84.26762819290161, + "agent_execution_time": 84.26762819290161, + "token_usage": { + "input_tokens": 108199, + "output_tokens": 610, + "total_tokens": 108809, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 192.07586979866028, + "agent_execution_time": 192.07586979866028, + "token_usage": { + "input_tokens": 456113, + "output_tokens": 1039, + "total_tokens": 457152, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 292.9134404659271, + "agent_execution_time": 292.9134404659271, + "token_usage": { + "input_tokens": 204877, + "output_tokens": 618, + "total_tokens": 205495, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 10.692319631576538, + "agent_execution_time": 10.692319631576538, + "token_usage": { + "input_tokens": 4307, + "output_tokens": 189, + "total_tokens": 4496, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 144.99, + "avg_input_tokens": 193374.0, + "avg_output_tokens": 614.0, + "avg_total_tokens": 193988.0, + "avg_turn_count": 7.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 59.010971546173096, + "agent_execution_time": 59.010971546173096, + "token_usage": { + "input_tokens": 1110184, + "output_tokens": 2823, + "total_tokens": 1113007, + "reasoning_tokens": 1952 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": true, + "execution_time": 76.2082691192627, + "agent_execution_time": 76.2082691192627, + "token_usage": { + "input_tokens": 1152880, + "output_tokens": 2697, + "total_tokens": 1155577, + "reasoning_tokens": 1848 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": true, + "execution_time": 74.61527252197266, + "agent_execution_time": 74.61527252197266, + "token_usage": { + "input_tokens": 1122447, + "output_tokens": 2244, + "total_tokens": 1124691, + "reasoning_tokens": 1413 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": true, + "execution_time": 233.45671105384827, + "agent_execution_time": 233.45671105384827, + "token_usage": { + "input_tokens": 1418334, + "output_tokens": 2465, + "total_tokens": 1420799, + "reasoning_tokens": 1578 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 110.82, + "avg_input_tokens": 1200961.2, + "avg_output_tokens": 2557.2, + "avg_total_tokens": 1203518.5, + "avg_turn_count": 15.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/security__rls_business_access.json b/task_results/security__rls_business_access.json index 5a976ba8..df4dda10 100644 --- a/task_results/security__rls_business_access.json +++ b/task_results/security__rls_business_access.json @@ -2,6 +2,73 @@ "task": "security__rls_business_access", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 601.8553755283356, + "agent_execution_time": 601.8553755283356, + "token_usage": { + "input_tokens": 553303, + "output_tokens": 9866, + "total_tokens": 563169, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + { + "run": "run-2", + "success": false, + "execution_time": 280.5147569179535, + "agent_execution_time": 280.5147569179535, + "token_usage": { + "input_tokens": 230594, + "output_tokens": 4059, + "total_tokens": 234653, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1091.5518026351929, + "agent_execution_time": 1091.5518026351929, + "token_usage": { + "input_tokens": 59195, + "output_tokens": 31414, + "total_tokens": 90609, + "reasoning_tokens": 27365 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 736.2105123996735, + "agent_execution_time": 736.2105123996735, + "token_usage": { + "input_tokens": 76180, + "output_tokens": 16699, + "total_tokens": 92879, + "reasoning_tokens": 10820 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 677.53, + "avg_input_tokens": 229818.0, + "avg_output_tokens": 15509.5, + "avg_total_tokens": 245327.5, + "avg_turn_count": 20.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 1202.568341255188, - "agent_execution_time": 1202.568341255188, - "token_usage": { - "input_tokens": 699229, - "output_tokens": 22522, - "total_tokens": 721751, - "reasoning_tokens": null - }, - "turn_count": 41 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 1202.57, - "avg_input_tokens": 699229.0, - "avg_output_tokens": 22522.0, - "avg_total_tokens": 721751.0, - "avg_turn_count": 41.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 230.73492789268494, + "agent_execution_time": 230.73492789268494, + "token_usage": { + "input_tokens": 480372, + "output_tokens": 13967, + "total_tokens": 494339, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": true, + "execution_time": 207.54959177970886, + "agent_execution_time": 207.54959177970886, + "token_usage": { + "input_tokens": 326409, + "output_tokens": 12702, + "total_tokens": 339111, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": true, + "execution_time": 182.66811847686768, + "agent_execution_time": 182.66811847686768, + "token_usage": { + "input_tokens": 409390, + "output_tokens": 9802, + "total_tokens": 419192, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": true, + "execution_time": 171.18537974357605, + "agent_execution_time": 171.18537974357605, + "token_usage": { + "input_tokens": 311090, + "output_tokens": 9247, + "total_tokens": 320337, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 198.03, + "avg_input_tokens": 381815.2, + "avg_output_tokens": 11429.5, + "avg_total_tokens": 393244.8, + "avg_turn_count": 21.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 299.8071575164795, + "agent_execution_time": 299.8071575164795, + "token_usage": { + "input_tokens": 206144, + "output_tokens": 5802, + "total_tokens": 211946, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": true, + "execution_time": 434.64122343063354, + "agent_execution_time": 434.64122343063354, + "token_usage": { + "input_tokens": 214011, + "output_tokens": 6160, + "total_tokens": 220171, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": true, + "execution_time": 188.7235472202301, + "agent_execution_time": 188.7235472202301, + "token_usage": { + "input_tokens": 224582, + "output_tokens": 6268, + "total_tokens": 230850, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 218.0106339454651, + "agent_execution_time": 218.0106339454651, + "token_usage": { + "input_tokens": 259696, + "output_tokens": 6795, + "total_tokens": 266491, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 285.3, + "avg_input_tokens": 226108.2, + "avg_output_tokens": 6256.2, + "avg_total_tokens": 232364.5, + "avg_turn_count": 22.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 254.88371181488037, + "agent_execution_time": 254.88371181488037, + "token_usage": { + "input_tokens": 205896, + "output_tokens": 4375, + "total_tokens": 210271, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": false, + "execution_time": 332.03830099105835, + "agent_execution_time": 332.03830099105835, + "token_usage": { + "input_tokens": 267193, + "output_tokens": 5243, + "total_tokens": 272436, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": false, + "execution_time": 293.414235830307, + "agent_execution_time": 293.414235830307, + "token_usage": { + "input_tokens": 189868, + "output_tokens": 4662, + "total_tokens": 194530, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.234999179840088, + "agent_execution_time": 8.234999179840088, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 222.14, + "avg_input_tokens": 165739.2, + "avg_output_tokens": 3570.0, + "avg_total_tokens": 169309.2, + "avg_turn_count": 17.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 54.152281522750854, + "agent_execution_time": 54.152281522750854, + "token_usage": { + "input_tokens": 103296, + "output_tokens": 6756, + "total_tokens": 110052, + "reasoning_tokens": 3199 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": true, + "execution_time": 89.85063052177429, + "agent_execution_time": 89.85063052177429, + "token_usage": { + "input_tokens": 56036, + "output_tokens": 7134, + "total_tokens": 63170, + "reasoning_tokens": 5240 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": true, + "execution_time": 79.84797501564026, + "agent_execution_time": 79.84797501564026, + "token_usage": { + "input_tokens": 147064, + "output_tokens": 5916, + "total_tokens": 152980, + "reasoning_tokens": 4262 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 110.54613542556763, + "agent_execution_time": 110.54613542556763, + "token_usage": { + "input_tokens": 471402, + "output_tokens": 6553, + "total_tokens": 477955, + "reasoning_tokens": 3542 + }, + "turn_count": 43 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 83.6, + "avg_input_tokens": 194449.5, + "avg_output_tokens": 6589.8, + "avg_total_tokens": 201039.2, + "avg_turn_count": 20.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/security__user_permission_audit.json b/task_results/security__user_permission_audit.json index 27c5e9a8..1a5dd17e 100644 --- a/task_results/security__user_permission_audit.json +++ b/task_results/security__user_permission_audit.json @@ -2,6 +2,73 @@ "task": "security__user_permission_audit", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 310.8364751338959, + "agent_execution_time": 310.8364751338959, + "token_usage": { + "input_tokens": 159466, + "output_tokens": 5560, + "total_tokens": 165026, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 314.9047932624817, + "agent_execution_time": 314.9047932624817, + "token_usage": { + "input_tokens": 240764, + "output_tokens": 5295, + "total_tokens": 246059, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1416.4049904346466, + "agent_execution_time": 1416.4049904346466, + "token_usage": { + "input_tokens": 142863, + "output_tokens": 38649, + "total_tokens": 181512, + "reasoning_tokens": 35302 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 158.8768002986908, + "agent_execution_time": 158.8768002986908, + "token_usage": { + "input_tokens": 7078, + "output_tokens": 4165, + "total_tokens": 11243, + "reasoning_tokens": 4129 + }, + "turn_count": 2 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 550.26, + "avg_input_tokens": 137542.8, + "avg_output_tokens": 13417.2, + "avg_total_tokens": 150960.0, + "avg_turn_count": 12.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 403.78105878829956, - "agent_execution_time": 403.78105878829956, - "token_usage": { - "input_tokens": 174179, - "output_tokens": 6917, - "total_tokens": 181096, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 403.78, - "avg_input_tokens": 174179.0, - "avg_output_tokens": 6917.0, - "avg_total_tokens": 181096.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 102.74184060096741, + "agent_execution_time": 102.74184060096741, + "token_usage": { + "input_tokens": 149227, + "output_tokens": 6250, + "total_tokens": 155477, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 114.31816673278809, + "agent_execution_time": 114.31816673278809, + "token_usage": { + "input_tokens": 185891, + "output_tokens": 6043, + "total_tokens": 191934, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": false, + "execution_time": 164.20190405845642, + "agent_execution_time": 164.20190405845642, + "token_usage": { + "input_tokens": 322751, + "output_tokens": 9400, + "total_tokens": 332151, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 135.47405791282654, + "agent_execution_time": 135.47405791282654, + "token_usage": { + "input_tokens": 271906, + "output_tokens": 6009, + "total_tokens": 277915, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 129.18, + "avg_input_tokens": 232443.8, + "avg_output_tokens": 6925.5, + "avg_total_tokens": 239369.2, + "avg_turn_count": 18.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 178.9361207485199, + "agent_execution_time": 178.9361207485199, + "token_usage": { + "input_tokens": 398130, + "output_tokens": 5274, + "total_tokens": 403404, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 226.35493350028992, + "agent_execution_time": 226.35493350028992, + "token_usage": { + "input_tokens": 42591, + "output_tokens": 1274, + "total_tokens": 43865, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 190.66862654685974, + "agent_execution_time": 190.66862654685974, + "token_usage": { + "input_tokens": 210807, + "output_tokens": 4114, + "total_tokens": 214921, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 140.01296734809875, + "agent_execution_time": 140.01296734809875, + "token_usage": { + "input_tokens": 159500, + "output_tokens": 5517, + "total_tokens": 165017, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 183.99, + "avg_input_tokens": 202757.0, + "avg_output_tokens": 4044.8, + "avg_total_tokens": 206801.8, + "avg_turn_count": 17.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 293.8720893859863, + "agent_execution_time": 293.8720893859863, + "token_usage": { + "input_tokens": 191825, + "output_tokens": 5986, + "total_tokens": 197811, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 224.15211606025696, + "agent_execution_time": 224.15211606025696, + "token_usage": { + "input_tokens": 128597, + "output_tokens": 3711, + "total_tokens": 132308, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 261.39845728874207, + "agent_execution_time": 261.39845728874207, + "token_usage": { + "input_tokens": 147157, + "output_tokens": 4468, + "total_tokens": 151625, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.308505773544312, + "agent_execution_time": 8.308505773544312, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 196.93, + "avg_input_tokens": 116894.8, + "avg_output_tokens": 3541.2, + "avg_total_tokens": 120436.0, + "avg_turn_count": 11.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 40.413169384002686, + "agent_execution_time": 40.413169384002686, + "token_usage": { + "input_tokens": 81237, + "output_tokens": 5574, + "total_tokens": 86811, + "reasoning_tokens": 4487 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 48.254390001297, + "agent_execution_time": 48.254390001297, + "token_usage": { + "input_tokens": 61618, + "output_tokens": 4853, + "total_tokens": 66471, + "reasoning_tokens": 3737 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 46.04936122894287, + "agent_execution_time": 46.04936122894287, + "token_usage": { + "input_tokens": 66601, + "output_tokens": 5599, + "total_tokens": 72200, + "reasoning_tokens": 4557 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 33.74569845199585, + "agent_execution_time": 33.74569845199585, + "token_usage": { + "input_tokens": 79851, + "output_tokens": 4185, + "total_tokens": 84036, + "reasoning_tokens": 2991 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 42.12, + "avg_input_tokens": 72326.8, + "avg_output_tokens": 5052.8, + "avg_total_tokens": 77379.5, + "avg_turn_count": 8.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/self_assessment__faq_column_layout.json b/task_results/self_assessment__faq_column_layout.json index 2ac216bc..d96ac76b 100644 --- a/task_results/self_assessment__faq_column_layout.json +++ b/task_results/self_assessment__faq_column_layout.json @@ -2,6 +2,73 @@ "task": "self_assessment__faq_column_layout", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 193.1650745868683, + "agent_execution_time": 193.1650745868683, + "token_usage": { + "input_tokens": 516625, + "output_tokens": 1811, + "total_tokens": 518436, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1562.039703130722, + "agent_execution_time": 1562.039703130722, + "token_usage": { + "input_tokens": 151669, + "output_tokens": 37060, + "total_tokens": 188729, + "reasoning_tokens": 35734 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 191.101984500885, + "agent_execution_time": 191.101984500885, + "token_usage": { + "input_tokens": 438906, + "output_tokens": 1939, + "total_tokens": 440845, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": true, + "execution_time": 1171.1003983020782, + "agent_execution_time": 1171.1003983020782, + "token_usage": { + "input_tokens": 174715, + "output_tokens": 27374, + "total_tokens": 202089, + "reasoning_tokens": 24938 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 779.35, + "avg_input_tokens": 320478.8, + "avg_output_tokens": 17046.0, + "avg_total_tokens": 337524.8, + "avg_turn_count": 11.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 214.57503175735474, - "agent_execution_time": 214.57503175735474, - "token_usage": { - "input_tokens": 294092, - "output_tokens": 2935, - "total_tokens": 297027, - "reasoning_tokens": null - }, - "turn_count": 11 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 214.58, - "avg_input_tokens": 294092.0, - "avg_output_tokens": 2935.0, - "avg_total_tokens": 297027.0, - "avg_turn_count": 11.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 109.74995875358582, + "agent_execution_time": 109.74995875358582, + "token_usage": { + "input_tokens": 255584, + "output_tokens": 4426, + "total_tokens": 260010, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 70.72846961021423, + "agent_execution_time": 70.72846961021423, + "token_usage": { + "input_tokens": 141898, + "output_tokens": 2968, + "total_tokens": 144866, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 118.35163760185242, + "agent_execution_time": 118.35163760185242, + "token_usage": { + "input_tokens": 507057, + "output_tokens": 3783, + "total_tokens": 510840, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 108.76008439064026, + "agent_execution_time": 108.76008439064026, + "token_usage": { + "input_tokens": 322068, + "output_tokens": 3631, + "total_tokens": 325699, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 101.9, + "avg_input_tokens": 306651.8, + "avg_output_tokens": 3702.0, + "avg_total_tokens": 310353.8, + "avg_turn_count": 9.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 132.00747561454773, + "agent_execution_time": 132.00747561454773, + "token_usage": { + "input_tokens": 585424, + "output_tokens": 1822, + "total_tokens": 587246, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": false, + "execution_time": 85.53575325012207, + "agent_execution_time": 85.53575325012207, + "token_usage": { + "input_tokens": 397865, + "output_tokens": 1783, + "total_tokens": 399648, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": true, + "execution_time": 244.30236291885376, + "agent_execution_time": 244.30236291885376, + "token_usage": { + "input_tokens": 395464, + "output_tokens": 2801, + "total_tokens": 398265, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 98.54328751564026, + "agent_execution_time": 98.54328751564026, + "token_usage": { + "input_tokens": 367330, + "output_tokens": 2850, + "total_tokens": 370180, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 140.1, + "avg_input_tokens": 436520.8, + "avg_output_tokens": 2314.0, + "avg_total_tokens": 438834.8, + "avg_turn_count": 17.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 336.40138387680054, + "agent_execution_time": 336.40138387680054, + "token_usage": { + "input_tokens": 973443, + "output_tokens": 3126, + "total_tokens": 976569, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-2", + "success": false, + "execution_time": 77.71289849281311, + "agent_execution_time": 77.71289849281311, + "token_usage": { + "input_tokens": 113533, + "output_tokens": 1094, + "total_tokens": 114627, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 200.61159801483154, + "agent_execution_time": 200.61159801483154, + "token_usage": { + "input_tokens": 631844, + "output_tokens": 1993, + "total_tokens": 633837, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 206.68745589256287, + "agent_execution_time": 206.68745589256287, + "token_usage": { + "input_tokens": 482174, + "output_tokens": 1804, + "total_tokens": 483978, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 205.35, + "avg_input_tokens": 550248.5, + "avg_output_tokens": 2004.2, + "avg_total_tokens": 552252.8, + "avg_turn_count": 17.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 80.69040131568909, + "agent_execution_time": 80.69040131568909, + "token_usage": { + "input_tokens": 465097, + "output_tokens": 5657, + "total_tokens": 470754, + "reasoning_tokens": 3970 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 81.31278705596924, + "agent_execution_time": 81.31278705596924, + "token_usage": { + "input_tokens": 300954, + "output_tokens": 4467, + "total_tokens": 305421, + "reasoning_tokens": 3541 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 75.64905261993408, + "agent_execution_time": 75.64905261993408, + "token_usage": { + "input_tokens": 437946, + "output_tokens": 5118, + "total_tokens": 443064, + "reasoning_tokens": 3982 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 80.1673743724823, + "agent_execution_time": 80.1673743724823, + "token_usage": { + "input_tokens": 522323, + "output_tokens": 6350, + "total_tokens": 528673, + "reasoning_tokens": 4941 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 79.45, + "avg_input_tokens": 431580.0, + "avg_output_tokens": 5398.0, + "avg_total_tokens": 436978.0, + "avg_turn_count": 16.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/self_assessment__hyperfocus_analysis_report.json b/task_results/self_assessment__hyperfocus_analysis_report.json index 72d1a595..7c1a7c49 100644 --- a/task_results/self_assessment__hyperfocus_analysis_report.json +++ b/task_results/self_assessment__hyperfocus_analysis_report.json @@ -2,6 +2,73 @@ "task": "self_assessment__hyperfocus_analysis_report", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 594.8983533382416, + "agent_execution_time": 594.8983533382416, + "token_usage": { + "input_tokens": 107073, + "output_tokens": 15837, + "total_tokens": 122910, + "reasoning_tokens": 11387 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1396.3053958415985, + "agent_execution_time": 1396.3053958415985, + "token_usage": { + "input_tokens": 132588, + "output_tokens": 34353, + "total_tokens": 166941, + "reasoning_tokens": 31282 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 299.3982877731323, + "agent_execution_time": 299.3982877731323, + "token_usage": { + "input_tokens": 397273, + "output_tokens": 5127, + "total_tokens": 402400, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 296.9515995979309, + "agent_execution_time": 296.9515995979309, + "token_usage": { + "input_tokens": 6806, + "output_tokens": 7675, + "total_tokens": 14481, + "reasoning_tokens": 7615 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 646.89, + "avg_input_tokens": 160935.0, + "avg_output_tokens": 15748.0, + "avg_total_tokens": 176683.0, + "avg_turn_count": 5.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 251.0410759449005, - "agent_execution_time": 251.0410759449005, - "token_usage": { - "input_tokens": 264610, - "output_tokens": 4542, - "total_tokens": 269152, - "reasoning_tokens": null - }, - "turn_count": 8 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 251.04, - "avg_input_tokens": 264610.0, - "avg_output_tokens": 4542.0, - "avg_total_tokens": 269152.0, - "avg_turn_count": 8.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 130.06600856781006, + "agent_execution_time": 130.06600856781006, + "token_usage": { + "input_tokens": 210179, + "output_tokens": 5463, + "total_tokens": 215642, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 146.61025142669678, + "agent_execution_time": 146.61025142669678, + "token_usage": { + "input_tokens": 624249, + "output_tokens": 7174, + "total_tokens": 631423, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 146.38673877716064, + "agent_execution_time": 146.38673877716064, + "token_usage": { + "input_tokens": 605507, + "output_tokens": 6515, + "total_tokens": 612022, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 149.1700623035431, + "agent_execution_time": 149.1700623035431, + "token_usage": { + "input_tokens": 578711, + "output_tokens": 6556, + "total_tokens": 585267, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 143.06, + "avg_input_tokens": 504661.5, + "avg_output_tokens": 6427.0, + "avg_total_tokens": 511088.5, + "avg_turn_count": 12.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 305.67374324798584, + "agent_execution_time": 305.67374324798584, + "token_usage": { + "input_tokens": 259608, + "output_tokens": 3642, + "total_tokens": 263250, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 86.78921341896057, + "agent_execution_time": 86.78921341896057, + "token_usage": { + "input_tokens": 157733, + "output_tokens": 2203, + "total_tokens": 159936, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 343.852254152298, + "agent_execution_time": 343.852254152298, + "token_usage": { + "input_tokens": 408216, + "output_tokens": 4806, + "total_tokens": 413022, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 162.0008807182312, + "agent_execution_time": 162.0008807182312, + "token_usage": { + "input_tokens": 566951, + "output_tokens": 5369, + "total_tokens": 572320, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 224.58, + "avg_input_tokens": 348127.0, + "avg_output_tokens": 4005.0, + "avg_total_tokens": 352132.0, + "avg_turn_count": 13.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 257.5691246986389, + "agent_execution_time": 257.5691246986389, + "token_usage": { + "input_tokens": 211537, + "output_tokens": 4692, + "total_tokens": 216229, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 158.84276294708252, + "agent_execution_time": 158.84276294708252, + "token_usage": { + "input_tokens": 262346, + "output_tokens": 2737, + "total_tokens": 265083, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 352.6119296550751, + "agent_execution_time": 352.6119296550751, + "token_usage": { + "input_tokens": 495782, + "output_tokens": 5423, + "total_tokens": 501205, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 331.8921048641205, + "agent_execution_time": 331.8921048641205, + "token_usage": { + "input_tokens": 836669, + "output_tokens": 4143, + "total_tokens": 840812, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 275.23, + "avg_input_tokens": 451583.5, + "avg_output_tokens": 4248.8, + "avg_total_tokens": 455832.2, + "avg_turn_count": 11.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 490.66569781303406, + "agent_execution_time": 490.66569781303406, + "token_usage": { + "input_tokens": 228757, + "output_tokens": 84023, + "total_tokens": 312780, + "reasoning_tokens": 5616 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 87.10163140296936, + "agent_execution_time": 87.10163140296936, + "token_usage": { + "input_tokens": 161990, + "output_tokens": 9092, + "total_tokens": 171082, + "reasoning_tokens": 7771 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 74.65599489212036, + "agent_execution_time": 74.65599489212036, + "token_usage": { + "input_tokens": 193103, + "output_tokens": 6113, + "total_tokens": 199216, + "reasoning_tokens": 5126 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 74.88243842124939, + "agent_execution_time": 74.88243842124939, + "token_usage": { + "input_tokens": 220460, + "output_tokens": 7749, + "total_tokens": 228209, + "reasoning_tokens": 7115 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 181.83, + "avg_input_tokens": 201077.5, + "avg_output_tokens": 26744.2, + "avg_total_tokens": 227821.8, + "avg_turn_count": 9.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/self_assessment__numbered_list_emojis.json b/task_results/self_assessment__numbered_list_emojis.json index 2064337b..2dbb853e 100644 --- a/task_results/self_assessment__numbered_list_emojis.json +++ b/task_results/self_assessment__numbered_list_emojis.json @@ -2,6 +2,73 @@ "task": "self_assessment__numbered_list_emojis", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 601.8822836875916, + "agent_execution_time": 601.8822836875916, + "token_usage": { + "input_tokens": 106289, + "output_tokens": 16755, + "total_tokens": 123044, + "reasoning_tokens": 14120 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 842.8231256008148, + "agent_execution_time": 842.8231256008148, + "token_usage": { + "input_tokens": 72386, + "output_tokens": 20149, + "total_tokens": 92535, + "reasoning_tokens": 18592 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 78.12506318092346, + "agent_execution_time": 78.12506318092346, + "token_usage": { + "input_tokens": 171086, + "output_tokens": 771, + "total_tokens": 171857, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 793.7379803657532, + "agent_execution_time": 793.7379803657532, + "token_usage": { + "input_tokens": 71480, + "output_tokens": 19866, + "total_tokens": 91346, + "reasoning_tokens": 18900 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 579.14, + "avg_input_tokens": 105310.2, + "avg_output_tokens": 14385.2, + "avg_total_tokens": 119695.5, + "avg_turn_count": 5.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 256.4135408401489, - "agent_execution_time": 256.4135408401489, - "token_usage": { - "input_tokens": 452760, - "output_tokens": 2723, - "total_tokens": 455483, - "reasoning_tokens": null - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 256.41, - "avg_input_tokens": 452760.0, - "avg_output_tokens": 2723.0, - "avg_total_tokens": 455483.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 104.32679319381714, + "agent_execution_time": 104.32679319381714, + "token_usage": { + "input_tokens": 296811, + "output_tokens": 3844, + "total_tokens": 300655, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 103.34714984893799, + "agent_execution_time": 103.34714984893799, + "token_usage": { + "input_tokens": 575570, + "output_tokens": 4214, + "total_tokens": 579784, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 144.49188137054443, + "agent_execution_time": 144.49188137054443, + "token_usage": { + "input_tokens": 1089486, + "output_tokens": 4932, + "total_tokens": 1094418, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 111.16063499450684, + "agent_execution_time": 111.16063499450684, + "token_usage": { + "input_tokens": 359493, + "output_tokens": 4728, + "total_tokens": 364221, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 115.83, + "avg_input_tokens": 580340.0, + "avg_output_tokens": 4429.5, + "avg_total_tokens": 584769.5, + "avg_turn_count": 11.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 31.017234086990356, + "agent_execution_time": 31.017234086990356, + "token_usage": { + "input_tokens": 77635, + "output_tokens": 287, + "total_tokens": 77922, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 25.62408208847046, + "agent_execution_time": 25.62408208847046, + "token_usage": { + "input_tokens": 77460, + "output_tokens": 222, + "total_tokens": 77682, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 511.9784278869629, + "agent_execution_time": 511.9784278869629, + "token_usage": { + "input_tokens": 1066005, + "output_tokens": 4877, + "total_tokens": 1070882, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": false, + "execution_time": 158.38496708869934, + "agent_execution_time": 158.38496708869934, + "token_usage": { + "input_tokens": 1117600, + "output_tokens": 4180, + "total_tokens": 1121780, + "reasoning_tokens": 0 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 181.75, + "avg_input_tokens": 584675.0, + "avg_output_tokens": 2391.5, + "avg_total_tokens": 587066.5, + "avg_turn_count": 14.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 403.21801567077637, + "agent_execution_time": 403.21801567077637, + "token_usage": { + "input_tokens": 1665948, + "output_tokens": 3235, + "total_tokens": 1669183, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-2", + "success": false, + "execution_time": 73.6966302394867, + "agent_execution_time": 73.6966302394867, + "token_usage": { + "input_tokens": 104814, + "output_tokens": 1474, + "total_tokens": 106288, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": true, + "execution_time": 382.89486956596375, + "agent_execution_time": 382.89486956596375, + "token_usage": { + "input_tokens": 1718346, + "output_tokens": 3796, + "total_tokens": 1722142, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-4", + "success": false, + "execution_time": 102.92528820037842, + "agent_execution_time": 102.92528820037842, + "token_usage": { + "input_tokens": 275552, + "output_tokens": 847, + "total_tokens": 276399, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 240.68, + "avg_input_tokens": 941165.0, + "avg_output_tokens": 2338.0, + "avg_total_tokens": 943503.0, + "avg_turn_count": 20.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 121.29287600517273, + "agent_execution_time": 121.29287600517273, + "token_usage": { + "input_tokens": 717522, + "output_tokens": 10009, + "total_tokens": 727531, + "reasoning_tokens": 7553 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 508.04330611228943, + "agent_execution_time": 508.04330611228943, + "token_usage": { + "input_tokens": 927654, + "output_tokens": 11825, + "total_tokens": 939479, + "reasoning_tokens": 9298 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 553.093966960907, + "agent_execution_time": 553.093966960907, + "token_usage": { + "input_tokens": 226597, + "output_tokens": 103687, + "total_tokens": 330284, + "reasoning_tokens": 8157 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 93.50998592376709, + "agent_execution_time": 93.50998592376709, + "token_usage": { + "input_tokens": 276583, + "output_tokens": 9546, + "total_tokens": 286129, + "reasoning_tokens": 7042 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 318.99, + "avg_input_tokens": 537089.0, + "avg_output_tokens": 33766.8, + "avg_total_tokens": 570855.8, + "avg_turn_count": 15.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping__advanced_product_analysis.json b/task_results/shopping__advanced_product_analysis.json index e16410dc..8cf41ec8 100644 --- a/task_results/shopping__advanced_product_analysis.json +++ b/task_results/shopping__advanced_product_analysis.json @@ -2,6 +2,73 @@ "task": "shopping__advanced_product_analysis", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 629.6949858665466, + "agent_execution_time": 629.6949858665466, + "token_usage": { + "input_tokens": 1040189, + "output_tokens": 1147, + "total_tokens": 1041336, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 275.84868359565735, + "agent_execution_time": 275.84868359565735, + "token_usage": { + "input_tokens": 947733, + "output_tokens": 1243, + "total_tokens": 948976, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1413.9752473831177, + "agent_execution_time": 1413.9752473831177, + "token_usage": { + "input_tokens": 542616, + "output_tokens": 18572, + "total_tokens": 561188, + "reasoning_tokens": 17857 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1111.8663656711578, + "agent_execution_time": 1111.8663656711578, + "token_usage": { + "input_tokens": 571768, + "output_tokens": 16929, + "total_tokens": 588697, + "reasoning_tokens": 16228 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 857.85, + "avg_input_tokens": 775576.5, + "avg_output_tokens": 9472.8, + "avg_total_tokens": 785049.2, + "avg_turn_count": 15.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 452.6573419570923, - "agent_execution_time": 452.6573419570923, - "token_usage": { - "input_tokens": 1438842, - "output_tokens": 3138, - "total_tokens": 1441980, - "reasoning_tokens": null - }, - "turn_count": 20 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 452.66, - "avg_input_tokens": 1438842.0, - "avg_output_tokens": 3138.0, - "avg_total_tokens": 1441980.0, - "avg_turn_count": 20.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 161.12943959236145, + "agent_execution_time": 161.12943959236145, + "token_usage": { + "input_tokens": 1485013, + "output_tokens": 3170, + "total_tokens": 1488183, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": true, + "execution_time": 157.22969889640808, + "agent_execution_time": 157.22969889640808, + "token_usage": { + "input_tokens": 1438632, + "output_tokens": 2688, + "total_tokens": 1441320, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": true, + "execution_time": 209.5884816646576, + "agent_execution_time": 209.5884816646576, + "token_usage": { + "input_tokens": 1411639, + "output_tokens": 2970, + "total_tokens": 1414609, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": true, + "execution_time": 300.6007089614868, + "agent_execution_time": 300.6007089614868, + "token_usage": { + "input_tokens": 1441580, + "output_tokens": 2757, + "total_tokens": 1444337, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 207.14, + "avg_input_tokens": 1444216.0, + "avg_output_tokens": 2896.2, + "avg_total_tokens": 1447112.2, + "avg_turn_count": 19.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 365.90656566619873, + "agent_execution_time": 365.90656566619873, + "token_usage": { + "input_tokens": 1084750, + "output_tokens": 3353, + "total_tokens": 1088103, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": true, + "execution_time": 288.08566522598267, + "agent_execution_time": 288.08566522598267, + "token_usage": { + "input_tokens": 965974, + "output_tokens": 3279, + "total_tokens": 969253, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": true, + "execution_time": 153.26812481880188, + "agent_execution_time": 153.26812481880188, + "token_usage": { + "input_tokens": 958162, + "output_tokens": 3112, + "total_tokens": 961274, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 271.8477392196655, + "agent_execution_time": 271.8477392196655, + "token_usage": { + "input_tokens": 1081798, + "output_tokens": 2438, + "total_tokens": 1084236, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 269.78, + "avg_input_tokens": 1022671.0, + "avg_output_tokens": 3045.5, + "avg_total_tokens": 1025716.5, + "avg_turn_count": 16.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 245.26102876663208, + "agent_execution_time": 245.26102876663208, + "token_usage": { + "input_tokens": 930637, + "output_tokens": 1242, + "total_tokens": 931879, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 243.48658752441406, + "agent_execution_time": 243.48658752441406, + "token_usage": { + "input_tokens": 891158, + "output_tokens": 1333, + "total_tokens": 892491, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 819.8968021869659, + "agent_execution_time": 819.8968021869659, + "token_usage": { + "input_tokens": 1002016, + "output_tokens": 2160, + "total_tokens": 1004176, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1232.4235219955444, + "agent_execution_time": 1232.4235219955444, + "token_usage": { + "input_tokens": 1378068, + "output_tokens": 3012, + "total_tokens": 1381080, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 635.27, + "avg_input_tokens": 1050469.8, + "avg_output_tokens": 1936.8, + "avg_total_tokens": 1052406.5, + "avg_turn_count": 18.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 71.9132034778595, + "agent_execution_time": 71.9132034778595, + "token_usage": { + "input_tokens": 900874, + "output_tokens": 2312, + "total_tokens": 903186, + "reasoning_tokens": 1610 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": true, + "execution_time": 76.39111614227295, + "agent_execution_time": 76.39111614227295, + "token_usage": { + "input_tokens": 1326885, + "output_tokens": 3656, + "total_tokens": 1330541, + "reasoning_tokens": 2802 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 63.126275300979614, + "agent_execution_time": 63.126275300979614, + "token_usage": { + "input_tokens": 820299, + "output_tokens": 2765, + "total_tokens": 823064, + "reasoning_tokens": 2029 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": true, + "execution_time": 67.79973006248474, + "agent_execution_time": 67.79973006248474, + "token_usage": { + "input_tokens": 819706, + "output_tokens": 2768, + "total_tokens": 822474, + "reasoning_tokens": 2087 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 69.81, + "avg_input_tokens": 966941.0, + "avg_output_tokens": 2875.2, + "avg_total_tokens": 969816.2, + "avg_turn_count": 14.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping__gaming_accessories_analysis.json b/task_results/shopping__gaming_accessories_analysis.json index 88eb212a..42c2d812 100644 --- a/task_results/shopping__gaming_accessories_analysis.json +++ b/task_results/shopping__gaming_accessories_analysis.json @@ -2,6 +2,73 @@ "task": "shopping__gaming_accessories_analysis", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 686.551858663559, + "agent_execution_time": 686.551858663559, + "token_usage": { + "input_tokens": 1733890, + "output_tokens": 2749, + "total_tokens": 1736639, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1752.1773190498352, + "agent_execution_time": 1752.1773190498352, + "token_usage": { + "input_tokens": 970153, + "output_tokens": 42333, + "total_tokens": 1012486, + "reasoning_tokens": 40926 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 45.972373723983765, + "agent_execution_time": 45.972373723983765, + "token_usage": { + "input_tokens": 4366, + "output_tokens": 1074, + "total_tokens": 5440, + "reasoning_tokens": 1032 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 514.1069738864899, + "agent_execution_time": 514.1069738864899, + "token_usage": { + "input_tokens": 105535, + "output_tokens": 13709, + "total_tokens": 119244, + "reasoning_tokens": 13490 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 749.7, + "avg_input_tokens": 703486.0, + "avg_output_tokens": 14966.2, + "avg_total_tokens": 718452.2, + "avg_turn_count": 11.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 869.9663565158844, - "agent_execution_time": 869.9663565158844, - "token_usage": { - "input_tokens": 3654616, - "output_tokens": 4436, - "total_tokens": 3659052, - "reasoning_tokens": null - }, - "turn_count": 37 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 869.97, - "avg_input_tokens": 3654616.0, - "avg_output_tokens": 4436.0, - "avg_total_tokens": 3659052.0, - "avg_turn_count": 37.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 217.76334762573242, + "agent_execution_time": 217.76334762573242, + "token_usage": { + "input_tokens": 2311853, + "output_tokens": 4268, + "total_tokens": 2316121, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 323.9741506576538, + "agent_execution_time": 323.9741506576538, + "token_usage": { + "input_tokens": 2851047, + "output_tokens": 4572, + "total_tokens": 2855619, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-3", + "success": false, + "execution_time": 357.5807902812958, + "agent_execution_time": 357.5807902812958, + "token_usage": { + "input_tokens": 2466419, + "output_tokens": 4105, + "total_tokens": 2470524, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-4", + "success": false, + "execution_time": 232.4150779247284, + "agent_execution_time": 232.4150779247284, + "token_usage": { + "input_tokens": 2342845, + "output_tokens": 4211, + "total_tokens": 2347056, + "reasoning_tokens": 0 + }, + "turn_count": 28 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 282.93, + "avg_input_tokens": 2493041.0, + "avg_output_tokens": 4289.0, + "avg_total_tokens": 2497330.0, + "avg_turn_count": 29.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 429.2561628818512, + "agent_execution_time": 429.2561628818512, + "token_usage": { + "input_tokens": 2495652, + "output_tokens": 3591, + "total_tokens": 2499243, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + { + "run": "run-2", + "success": true, + "execution_time": 319.4610013961792, + "agent_execution_time": 319.4610013961792, + "token_usage": { + "input_tokens": 1938428, + "output_tokens": 4379, + "total_tokens": 1942807, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-3", + "success": true, + "execution_time": 280.0683240890503, + "agent_execution_time": 280.0683240890503, + "token_usage": { + "input_tokens": 1839114, + "output_tokens": 5494, + "total_tokens": 1844608, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": true, + "execution_time": 473.11012864112854, + "agent_execution_time": 473.11012864112854, + "token_usage": { + "input_tokens": 2664906, + "output_tokens": 5481, + "total_tokens": 2670387, + "reasoning_tokens": 0 + }, + "turn_count": 33 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 375.47, + "avg_input_tokens": 2234525.0, + "avg_output_tokens": 4736.2, + "avg_total_tokens": 2239261.2, + "avg_turn_count": 29.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 419.0592272281647, + "agent_execution_time": 419.0592272281647, + "token_usage": { + "input_tokens": 2072505, + "output_tokens": 3585, + "total_tokens": 2076090, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 416.41197776794434, + "agent_execution_time": 416.41197776794434, + "token_usage": { + "input_tokens": 2309633, + "output_tokens": 2960, + "total_tokens": 2312593, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-3", + "success": false, + "execution_time": 932.1683664321899, + "agent_execution_time": 932.1683664321899, + "token_usage": { + "input_tokens": 1363231, + "output_tokens": 3914, + "total_tokens": 1367145, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 67.24101996421814, + "agent_execution_time": 67.24101996421814, + "token_usage": { + "input_tokens": 39376, + "output_tokens": 882, + "total_tokens": 40258, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 458.72, + "avg_input_tokens": 1446186.2, + "avg_output_tokens": 2835.2, + "avg_total_tokens": 1449021.5, + "avg_turn_count": 19.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 113.50308537483215, + "agent_execution_time": 113.50308537483215, + "token_usage": { + "input_tokens": 1378782, + "output_tokens": 3499, + "total_tokens": 1382281, + "reasoning_tokens": 2346 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": true, + "execution_time": 169.1634442806244, + "agent_execution_time": 169.1634442806244, + "token_usage": { + "input_tokens": 2541389, + "output_tokens": 5463, + "total_tokens": 2546852, + "reasoning_tokens": 4037 + }, + "turn_count": 31 + }, + { + "run": "run-3", + "success": true, + "execution_time": 104.62143349647522, + "agent_execution_time": 104.62143349647522, + "token_usage": { + "input_tokens": 1663069, + "output_tokens": 4436, + "total_tokens": 1667505, + "reasoning_tokens": 2992 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 106.61391472816467, + "agent_execution_time": 106.61391472816467, + "token_usage": { + "input_tokens": 1581509, + "output_tokens": 4218, + "total_tokens": 1585727, + "reasoning_tokens": 2985 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 123.48, + "avg_input_tokens": 1791187.2, + "avg_output_tokens": 4404.0, + "avg_total_tokens": 1795591.2, + "avg_turn_count": 25.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping__health_routine_optimization.json b/task_results/shopping__health_routine_optimization.json index ab0fe949..3ba364b6 100644 --- a/task_results/shopping__health_routine_optimization.json +++ b/task_results/shopping__health_routine_optimization.json @@ -2,6 +2,73 @@ "task": "shopping__health_routine_optimization", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 748.5469524860382, + "agent_execution_time": 748.5469524860382, + "token_usage": { + "input_tokens": 1285621, + "output_tokens": 1653, + "total_tokens": 1287274, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1048.2380719184875, + "agent_execution_time": 1048.2380719184875, + "token_usage": { + "input_tokens": 513984, + "output_tokens": 25598, + "total_tokens": 539582, + "reasoning_tokens": 24864 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 2299.080590248108, + "agent_execution_time": 2299.080590248108, + "token_usage": { + "input_tokens": 1002161, + "output_tokens": 33398, + "total_tokens": 1035559, + "reasoning_tokens": 32389 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1483.6415514945984, + "agent_execution_time": 1483.6415514945984, + "token_usage": { + "input_tokens": 911088, + "output_tokens": 26239, + "total_tokens": 937327, + "reasoning_tokens": 25316 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1394.88, + "avg_input_tokens": 928213.5, + "avg_output_tokens": 21722.0, + "avg_total_tokens": 949935.5, + "avg_turn_count": 16.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 516.1689138412476, - "agent_execution_time": 516.1689138412476, - "token_usage": { - "input_tokens": 1719919, - "output_tokens": 3654, - "total_tokens": 1723573, - "reasoning_tokens": null - }, - "turn_count": 23 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 516.17, - "avg_input_tokens": 1719919.0, - "avg_output_tokens": 3654.0, - "avg_total_tokens": 1723573.0, - "avg_turn_count": 23.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 198.16809844970703, + "agent_execution_time": 198.16809844970703, + "token_usage": { + "input_tokens": 1864471, + "output_tokens": 3945, + "total_tokens": 1868416, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-2", + "success": true, + "execution_time": 260.1745903491974, + "agent_execution_time": 260.1745903491974, + "token_usage": { + "input_tokens": 2348968, + "output_tokens": 4281, + "total_tokens": 2353249, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-3", + "success": true, + "execution_time": 257.822390794754, + "agent_execution_time": 257.822390794754, + "token_usage": { + "input_tokens": 1649060, + "output_tokens": 3450, + "total_tokens": 1652510, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": true, + "execution_time": 202.98289275169373, + "agent_execution_time": 202.98289275169373, + "token_usage": { + "input_tokens": 1962549, + "output_tokens": 4216, + "total_tokens": 1966765, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 229.79, + "avg_input_tokens": 1956262.0, + "avg_output_tokens": 3973.0, + "avg_total_tokens": 1960235.0, + "avg_turn_count": 23.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 513.5799055099487, + "agent_execution_time": 513.5799055099487, + "token_usage": { + "input_tokens": 1541957, + "output_tokens": 5384, + "total_tokens": 1547341, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 417.6526165008545, + "agent_execution_time": 417.6526165008545, + "token_usage": { + "input_tokens": 1817885, + "output_tokens": 5266, + "total_tokens": 1823151, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": false, + "execution_time": 315.5167520046234, + "agent_execution_time": 315.5167520046234, + "token_usage": { + "input_tokens": 1616482, + "output_tokens": 5642, + "total_tokens": 1622124, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 452.0021958351135, + "agent_execution_time": 452.0021958351135, + "token_usage": { + "input_tokens": 1405861, + "output_tokens": 4614, + "total_tokens": 1410475, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 424.69, + "avg_input_tokens": 1595546.2, + "avg_output_tokens": 5226.5, + "avg_total_tokens": 1600772.8, + "avg_turn_count": 21.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 263.12338614463806, + "agent_execution_time": 263.12338614463806, + "token_usage": { + "input_tokens": 992238, + "output_tokens": 1455, + "total_tokens": 993693, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 303.2782862186432, + "agent_execution_time": 303.2782862186432, + "token_usage": { + "input_tokens": 1426931, + "output_tokens": 1563, + "total_tokens": 1428494, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 789.2739129066467, + "agent_execution_time": 789.2739129066467, + "token_usage": { + "input_tokens": 1433901, + "output_tokens": 1234, + "total_tokens": 1435135, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1412.375020980835, + "agent_execution_time": 1412.375020980835, + "token_usage": { + "input_tokens": 1635582, + "output_tokens": 2448, + "total_tokens": 1638030, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 692.01, + "avg_input_tokens": 1372163.0, + "avg_output_tokens": 1675.0, + "avg_total_tokens": 1373838.0, + "avg_turn_count": 20.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 88.40928888320923, + "agent_execution_time": 88.40928888320923, + "token_usage": { + "input_tokens": 1071847, + "output_tokens": 3604, + "total_tokens": 1075451, + "reasoning_tokens": 2693 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": true, + "execution_time": 103.08896255493164, + "agent_execution_time": 103.08896255493164, + "token_usage": { + "input_tokens": 1276853, + "output_tokens": 4802, + "total_tokens": 1281655, + "reasoning_tokens": 3802 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": true, + "execution_time": 86.69021010398865, + "agent_execution_time": 86.69021010398865, + "token_usage": { + "input_tokens": 1152264, + "output_tokens": 3728, + "total_tokens": 1155992, + "reasoning_tokens": 2727 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": true, + "execution_time": 88.08992671966553, + "agent_execution_time": 88.08992671966553, + "token_usage": { + "input_tokens": 1056042, + "output_tokens": 3398, + "total_tokens": 1059440, + "reasoning_tokens": 2386 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 91.57, + "avg_input_tokens": 1139251.5, + "avg_output_tokens": 3883.0, + "avg_total_tokens": 1143134.5, + "avg_turn_count": 17.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping__holiday_baking_competition.json b/task_results/shopping__holiday_baking_competition.json index 131fdfcb..dea0f85e 100644 --- a/task_results/shopping__holiday_baking_competition.json +++ b/task_results/shopping__holiday_baking_competition.json @@ -2,6 +2,73 @@ "task": "shopping__holiday_baking_competition", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 669.3894865512848, + "agent_execution_time": 669.3894865512848, + "token_usage": { + "input_tokens": 2034502, + "output_tokens": 1938, + "total_tokens": 2036440, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-2", + "success": false, + "execution_time": 646.8684825897217, + "agent_execution_time": 646.8684825897217, + "token_usage": { + "input_tokens": 220907, + "output_tokens": 15861, + "total_tokens": 236768, + "reasoning_tokens": 14856 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1921.1719779968262, + "agent_execution_time": 1921.1719779968262, + "token_usage": { + "input_tokens": 892104, + "output_tokens": 28581, + "total_tokens": 920685, + "reasoning_tokens": 27655 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1431.1701622009277, + "agent_execution_time": 1431.1701622009277, + "token_usage": { + "input_tokens": 906297, + "output_tokens": 31203, + "total_tokens": 937500, + "reasoning_tokens": 30256 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1167.15, + "avg_input_tokens": 1013452.5, + "avg_output_tokens": 19395.8, + "avg_total_tokens": 1032848.2, + "avg_turn_count": 16.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 647.6959841251373, - "agent_execution_time": 647.6959841251373, - "token_usage": { - "input_tokens": 2520129, - "output_tokens": 3457, - "total_tokens": 2523586, - "reasoning_tokens": null - }, - "turn_count": 26 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 647.7, - "avg_input_tokens": 2520129.0, - "avg_output_tokens": 3457.0, - "avg_total_tokens": 2523586.0, - "avg_turn_count": 26.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 208.97467589378357, + "agent_execution_time": 208.97467589378357, + "token_usage": { + "input_tokens": 2569627, + "output_tokens": 3522, + "total_tokens": 2573149, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": false, + "execution_time": 278.1035141944885, + "agent_execution_time": 278.1035141944885, + "token_usage": { + "input_tokens": 2491902, + "output_tokens": 4163, + "total_tokens": 2496065, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": false, + "execution_time": 365.9616105556488, + "agent_execution_time": 365.9616105556488, + "token_usage": { + "input_tokens": 2586401, + "output_tokens": 3482, + "total_tokens": 2589883, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-4", + "success": false, + "execution_time": 298.0338854789734, + "agent_execution_time": 298.0338854789734, + "token_usage": { + "input_tokens": 2449369, + "output_tokens": 4732, + "total_tokens": 2454101, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 287.77, + "avg_input_tokens": 2524324.8, + "avg_output_tokens": 3974.8, + "avg_total_tokens": 2528299.5, + "avg_turn_count": 25.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 830.770646572113, + "agent_execution_time": 830.770646572113, + "token_usage": { + "input_tokens": 2290402, + "output_tokens": 6994, + "total_tokens": 2297396, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 282.2895929813385, + "agent_execution_time": 282.2895929813385, + "token_usage": { + "input_tokens": 1602840, + "output_tokens": 5305, + "total_tokens": 1608145, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 456.966237783432, + "agent_execution_time": 456.966237783432, + "token_usage": { + "input_tokens": 2742848, + "output_tokens": 3090, + "total_tokens": 2745938, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-4", + "success": false, + "execution_time": 647.4424140453339, + "agent_execution_time": 647.4424140453339, + "token_usage": { + "input_tokens": 1653402, + "output_tokens": 3781, + "total_tokens": 1657183, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 554.37, + "avg_input_tokens": 2072373.0, + "avg_output_tokens": 4792.5, + "avg_total_tokens": 2077165.5, + "avg_turn_count": 25.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 366.3496813774109, + "agent_execution_time": 366.3496813774109, + "token_usage": { + "input_tokens": 1978847, + "output_tokens": 2446, + "total_tokens": 1981293, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-2", + "success": false, + "execution_time": 321.6748421192169, + "agent_execution_time": 321.6748421192169, + "token_usage": { + "input_tokens": 1609481, + "output_tokens": 1812, + "total_tokens": 1611293, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": false, + "execution_time": 701.5529799461365, + "agent_execution_time": 701.5529799461365, + "token_usage": { + "input_tokens": 1228856, + "output_tokens": 1515, + "total_tokens": 1230371, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 19.334036350250244, + "agent_execution_time": 19.334036350250244, + "token_usage": { + "input_tokens": 4383, + "output_tokens": 372, + "total_tokens": 4755, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 352.23, + "avg_input_tokens": 1205391.8, + "avg_output_tokens": 1536.2, + "avg_total_tokens": 1206928.0, + "avg_turn_count": 18.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 232.6989462375641, + "agent_execution_time": 232.6989462375641, + "token_usage": { + "input_tokens": 4272955, + "output_tokens": 7613, + "total_tokens": 4280568, + "reasoning_tokens": 5721 + }, + "turn_count": 36 + }, + { + "run": "run-2", + "success": true, + "execution_time": 210.50627279281616, + "agent_execution_time": 210.50627279281616, + "token_usage": { + "input_tokens": 3804506, + "output_tokens": 10585, + "total_tokens": 3815091, + "reasoning_tokens": 8147 + }, + "turn_count": 34 + }, + { + "run": "run-3", + "success": false, + "execution_time": 200.19812726974487, + "agent_execution_time": 200.19812726974487, + "token_usage": { + "input_tokens": 4302778, + "output_tokens": 10271, + "total_tokens": 4313049, + "reasoning_tokens": 8285 + }, + "turn_count": 35 + }, + { + "run": "run-4", + "success": false, + "execution_time": 155.43351340293884, + "agent_execution_time": 155.43351340293884, + "token_usage": { + "input_tokens": 2477101, + "output_tokens": 7072, + "total_tokens": 2484173, + "reasoning_tokens": 5675 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 199.71, + "avg_input_tokens": 3714335.0, + "avg_output_tokens": 8885.2, + "avg_total_tokens": 3723220.2, + "avg_turn_count": 31.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping__multi_category_budget_analysis.json b/task_results/shopping__multi_category_budget_analysis.json index ad43528e..4c1700b8 100644 --- a/task_results/shopping__multi_category_budget_analysis.json +++ b/task_results/shopping__multi_category_budget_analysis.json @@ -2,6 +2,73 @@ "task": "shopping__multi_category_budget_analysis", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 518.6517353057861, + "agent_execution_time": 518.6517353057861, + "token_usage": { + "input_tokens": 1350699, + "output_tokens": 1276, + "total_tokens": 1351975, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1072.4663262367249, + "agent_execution_time": 1072.4663262367249, + "token_usage": { + "input_tokens": 725787, + "output_tokens": 24569, + "total_tokens": 750356, + "reasoning_tokens": 23899 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 2045.283864736557, + "agent_execution_time": 2045.283864736557, + "token_usage": { + "input_tokens": 922923, + "output_tokens": 28792, + "total_tokens": 951715, + "reasoning_tokens": 27915 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1203.1308624744415, + "agent_execution_time": 1203.1308624744415, + "token_usage": { + "input_tokens": 719822, + "output_tokens": 30385, + "total_tokens": 750207, + "reasoning_tokens": 28955 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1209.88, + "avg_input_tokens": 929807.8, + "avg_output_tokens": 21255.5, + "avg_total_tokens": 951063.2, + "avg_turn_count": 16.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 690.4746689796448, - "agent_execution_time": 690.4746689796448, - "token_usage": { - "input_tokens": 2680142, - "output_tokens": 3645, - "total_tokens": 2683787, - "reasoning_tokens": null - }, - "turn_count": 27 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 690.47, - "avg_input_tokens": 2680142.0, - "avg_output_tokens": 3645.0, - "avg_total_tokens": 2683787.0, - "avg_turn_count": 27.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 211.2567048072815, + "agent_execution_time": 211.2567048072815, + "token_usage": { + "input_tokens": 2511826, + "output_tokens": 3745, + "total_tokens": 2515571, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": false, + "execution_time": 267.4960823059082, + "agent_execution_time": 267.4960823059082, + "token_usage": { + "input_tokens": 2592236, + "output_tokens": 3928, + "total_tokens": 2596164, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-3", + "success": false, + "execution_time": 188.2953622341156, + "agent_execution_time": 188.2953622341156, + "token_usage": { + "input_tokens": 2229840, + "output_tokens": 3706, + "total_tokens": 2233546, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 268.89570021629333, + "agent_execution_time": 268.89570021629333, + "token_usage": { + "input_tokens": 2467668, + "output_tokens": 3609, + "total_tokens": 2471277, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 233.99, + "avg_input_tokens": 2450392.5, + "avg_output_tokens": 3747.0, + "avg_total_tokens": 2454139.5, + "avg_turn_count": 25.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 793.4901113510132, + "agent_execution_time": 793.4901113510132, + "token_usage": { + "input_tokens": 2580076, + "output_tokens": 5897, + "total_tokens": 2585973, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-2", + "success": false, + "execution_time": 393.21907353401184, + "agent_execution_time": 393.21907353401184, + "token_usage": { + "input_tokens": 2040006, + "output_tokens": 5397, + "total_tokens": 2045403, + "reasoning_tokens": 0 + }, + "turn_count": 25 + }, + { + "run": "run-3", + "success": false, + "execution_time": 429.7153789997101, + "agent_execution_time": 429.7153789997101, + "token_usage": { + "input_tokens": 2493330, + "output_tokens": 5606, + "total_tokens": 2498936, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-4", + "success": false, + "execution_time": 968.8836879730225, + "agent_execution_time": 968.8836879730225, + "token_usage": { + "input_tokens": 1507110, + "output_tokens": 3525, + "total_tokens": 1510635, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 646.33, + "avg_input_tokens": 2155130.5, + "avg_output_tokens": 5106.2, + "avg_total_tokens": 2160236.8, + "avg_turn_count": 26.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 236.3616054058075, + "agent_execution_time": 236.3616054058075, + "token_usage": { + "input_tokens": 1157592, + "output_tokens": 1177, + "total_tokens": 1158769, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 266.07610630989075, + "agent_execution_time": 266.07610630989075, + "token_usage": { + "input_tokens": 1335356, + "output_tokens": 1261, + "total_tokens": 1336617, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": false, + "execution_time": 14.579027891159058, + "agent_execution_time": 14.579027891159058, + "token_usage": { + "input_tokens": 4477, + "output_tokens": 266, + "total_tokens": 4743, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 948.7483124732971, + "agent_execution_time": 948.7483124732971, + "token_usage": { + "input_tokens": 1330972, + "output_tokens": 1332, + "total_tokens": 1332304, + "reasoning_tokens": 0 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 366.44, + "avg_input_tokens": 957099.2, + "avg_output_tokens": 1009.0, + "avg_total_tokens": 958108.2, + "avg_turn_count": 15.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 251.94581246376038, + "agent_execution_time": 251.94581246376038, + "token_usage": { + "input_tokens": 6628911, + "output_tokens": 10458, + "total_tokens": 6639369, + "reasoning_tokens": 8158 + }, + "turn_count": 49 + }, + { + "run": "run-2", + "success": false, + "execution_time": 194.21247267723083, + "agent_execution_time": 194.21247267723083, + "token_usage": { + "input_tokens": 3190375, + "output_tokens": 10333, + "total_tokens": 3200708, + "reasoning_tokens": 8734 + }, + "turn_count": 32 + }, + { + "run": "run-3", + "success": false, + "execution_time": 182.10172986984253, + "agent_execution_time": 182.10172986984253, + "token_usage": { + "input_tokens": 4155619, + "output_tokens": 10557, + "total_tokens": 4166176, + "reasoning_tokens": 9018 + }, + "turn_count": 35 + }, + { + "run": "run-4", + "success": false, + "execution_time": 176.0252137184143, + "agent_execution_time": 176.0252137184143, + "token_usage": { + "input_tokens": 2255318, + "output_tokens": 7567, + "total_tokens": 2262885, + "reasoning_tokens": 6433 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 201.07, + "avg_input_tokens": 4057555.8, + "avg_output_tokens": 9728.8, + "avg_total_tokens": 4067284.5, + "avg_turn_count": 35.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping__printer_keyboard_search.json b/task_results/shopping__printer_keyboard_search.json index 8965f417..b273a602 100644 --- a/task_results/shopping__printer_keyboard_search.json +++ b/task_results/shopping__printer_keyboard_search.json @@ -2,6 +2,73 @@ "task": "shopping__printer_keyboard_search", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 251.4486665725708, + "agent_execution_time": 251.4486665725708, + "token_usage": { + "input_tokens": 1028158, + "output_tokens": 1516, + "total_tokens": 1029674, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 806.9798963069916, + "agent_execution_time": 806.9798963069916, + "token_usage": { + "input_tokens": 558587, + "output_tokens": 18650, + "total_tokens": 577237, + "reasoning_tokens": 18014 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1499.2226073741913, + "agent_execution_time": 1499.2226073741913, + "token_usage": { + "input_tokens": 449916, + "output_tokens": 26368, + "total_tokens": 476284, + "reasoning_tokens": 25624 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 306.85438871383667, + "agent_execution_time": 306.85438871383667, + "token_usage": { + "input_tokens": 94908, + "output_tokens": 7599, + "total_tokens": 102507, + "reasoning_tokens": 7356 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 716.13, + "avg_input_tokens": 532892.2, + "avg_output_tokens": 13533.2, + "avg_total_tokens": 546425.5, + "avg_turn_count": 11.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 216.40940713882446, - "agent_execution_time": 216.40940713882446, - "token_usage": { - "input_tokens": 474583, - "output_tokens": 1824, - "total_tokens": 476407, - "reasoning_tokens": null - }, - "turn_count": 10 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 216.41, - "avg_input_tokens": 474583.0, - "avg_output_tokens": 1824.0, - "avg_total_tokens": 476407.0, - "avg_turn_count": 10.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 133.2373776435852, + "agent_execution_time": 133.2373776435852, + "token_usage": { + "input_tokens": 839378, + "output_tokens": 2754, + "total_tokens": 842132, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 171.02756547927856, + "agent_execution_time": 171.02756547927856, + "token_usage": { + "input_tokens": 1193888, + "output_tokens": 3251, + "total_tokens": 1197139, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 161.84360098838806, + "agent_execution_time": 161.84360098838806, + "token_usage": { + "input_tokens": 1478641, + "output_tokens": 3198, + "total_tokens": 1481839, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 154.02386164665222, + "agent_execution_time": 154.02386164665222, + "token_usage": { + "input_tokens": 1312878, + "output_tokens": 2796, + "total_tokens": 1315674, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 155.03, + "avg_input_tokens": 1206196.2, + "avg_output_tokens": 2999.8, + "avg_total_tokens": 1209196.0, + "avg_turn_count": 17.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 547.8928933143616, + "agent_execution_time": 547.8928933143616, + "token_usage": { + "input_tokens": 1286864, + "output_tokens": 5786, + "total_tokens": 1292650, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": true, + "execution_time": 110.55178308486938, + "agent_execution_time": 110.55178308486938, + "token_usage": { + "input_tokens": 384916, + "output_tokens": 2515, + "total_tokens": 387431, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 259.6538293361664, + "agent_execution_time": 259.6538293361664, + "token_usage": { + "input_tokens": 1065550, + "output_tokens": 3600, + "total_tokens": 1069150, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 133.53703093528748, + "agent_execution_time": 133.53703093528748, + "token_usage": { + "input_tokens": 34888, + "output_tokens": 463, + "total_tokens": 35351, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 262.91, + "avg_input_tokens": 693054.5, + "avg_output_tokens": 3091.0, + "avg_total_tokens": 696145.5, + "avg_turn_count": 12.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 233.01101899147034, + "agent_execution_time": 233.01101899147034, + "token_usage": { + "input_tokens": 792882, + "output_tokens": 1899, + "total_tokens": 794781, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 209.66281485557556, + "agent_execution_time": 209.66281485557556, + "token_usage": { + "input_tokens": 767609, + "output_tokens": 1392, + "total_tokens": 769001, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 11.628572702407837, + "agent_execution_time": 11.628572702407837, + "token_usage": { + "input_tokens": 4220, + "output_tokens": 214, + "total_tokens": 4434, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 11.900094270706177, + "agent_execution_time": 11.900094270706177, + "token_usage": { + "input_tokens": 4220, + "output_tokens": 250, + "total_tokens": 4470, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 116.55, + "avg_input_tokens": 392232.8, + "avg_output_tokens": 938.8, + "avg_total_tokens": 393171.5, + "avg_turn_count": 8.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 64.0178050994873, + "agent_execution_time": 64.0178050994873, + "token_usage": { + "input_tokens": 505777, + "output_tokens": 2920, + "total_tokens": 508697, + "reasoning_tokens": 2465 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 65.19870638847351, + "agent_execution_time": 65.19870638847351, + "token_usage": { + "input_tokens": 663763, + "output_tokens": 3066, + "total_tokens": 666829, + "reasoning_tokens": 2300 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": true, + "execution_time": 98.66196751594543, + "agent_execution_time": 98.66196751594543, + "token_usage": { + "input_tokens": 1325602, + "output_tokens": 4775, + "total_tokens": 1330377, + "reasoning_tokens": 3774 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 96.52966380119324, + "agent_execution_time": 96.52966380119324, + "token_usage": { + "input_tokens": 1180344, + "output_tokens": 4154, + "total_tokens": 1184498, + "reasoning_tokens": 3410 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 81.1, + "avg_input_tokens": 918871.5, + "avg_output_tokens": 3728.8, + "avg_total_tokens": 922600.2, + "avg_turn_count": 14.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping__running_shoes_purchase.json b/task_results/shopping__running_shoes_purchase.json index 0318748e..2ae88e3e 100644 --- a/task_results/shopping__running_shoes_purchase.json +++ b/task_results/shopping__running_shoes_purchase.json @@ -2,6 +2,73 @@ "task": "shopping__running_shoes_purchase", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 185.48341250419617, + "agent_execution_time": 185.48341250419617, + "token_usage": { + "input_tokens": 595723, + "output_tokens": 1669, + "total_tokens": 597392, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 2192.8332035541534, + "agent_execution_time": 2192.8332035541534, + "token_usage": { + "input_tokens": 876020, + "output_tokens": 51393, + "total_tokens": 927413, + "reasoning_tokens": 50579 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1892.1069226264954, + "agent_execution_time": 1892.1069226264954, + "token_usage": { + "input_tokens": 577998, + "output_tokens": 34111, + "total_tokens": 612109, + "reasoning_tokens": 33387 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1839.153243303299, + "agent_execution_time": 1839.153243303299, + "token_usage": { + "input_tokens": 988076, + "output_tokens": 46598, + "total_tokens": 1034674, + "reasoning_tokens": 45722 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1527.39, + "avg_input_tokens": 759454.2, + "avg_output_tokens": 33442.8, + "avg_total_tokens": 792897.0, + "avg_turn_count": 14.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 258.12507224082947, - "agent_execution_time": 258.12507224082947, - "token_usage": { - "input_tokens": 707798, - "output_tokens": 2069, - "total_tokens": 709867, - "reasoning_tokens": null - }, - "turn_count": 13 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 258.13, - "avg_input_tokens": 707798.0, - "avg_output_tokens": 2069.0, - "avg_total_tokens": 709867.0, - "avg_turn_count": 13.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 138.48571109771729, + "agent_execution_time": 138.48571109771729, + "token_usage": { + "input_tokens": 1231565, + "output_tokens": 2616, + "total_tokens": 1234181, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 168.50810718536377, + "agent_execution_time": 168.50810718536377, + "token_usage": { + "input_tokens": 1007709, + "output_tokens": 3169, + "total_tokens": 1010878, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 110.85117816925049, + "agent_execution_time": 110.85117816925049, + "token_usage": { + "input_tokens": 793148, + "output_tokens": 2363, + "total_tokens": 795511, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 138.63545441627502, + "agent_execution_time": 138.63545441627502, + "token_usage": { + "input_tokens": 1399851, + "output_tokens": 2543, + "total_tokens": 1402394, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 139.12, + "avg_input_tokens": 1108068.2, + "avg_output_tokens": 2672.8, + "avg_total_tokens": 1110741.0, + "avg_turn_count": 15.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 278.5891230106354, + "agent_execution_time": 278.5891230106354, + "token_usage": { + "input_tokens": 358437, + "output_tokens": 3075, + "total_tokens": 361512, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 136.38987636566162, + "agent_execution_time": 136.38987636566162, + "token_usage": { + "input_tokens": 531715, + "output_tokens": 2361, + "total_tokens": 534076, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 139.0381636619568, + "agent_execution_time": 139.0381636619568, + "token_usage": { + "input_tokens": 344980, + "output_tokens": 2858, + "total_tokens": 347838, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 122.1661901473999, + "agent_execution_time": 122.1661901473999, + "token_usage": { + "input_tokens": 72262, + "output_tokens": 465, + "total_tokens": 72727, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 169.05, + "avg_input_tokens": 326848.5, + "avg_output_tokens": 2189.8, + "avg_total_tokens": 329038.2, + "avg_turn_count": 9.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 190.89328169822693, + "agent_execution_time": 190.89328169822693, + "token_usage": { + "input_tokens": 597536, + "output_tokens": 1830, + "total_tokens": 599366, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 176.13676643371582, + "agent_execution_time": 176.13676643371582, + "token_usage": { + "input_tokens": 539729, + "output_tokens": 1411, + "total_tokens": 541140, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 13.315120220184326, + "agent_execution_time": 13.315120220184326, + "token_usage": { + "input_tokens": 4192, + "output_tokens": 229, + "total_tokens": 4421, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 681.7983357906342, + "agent_execution_time": 681.7983357906342, + "token_usage": { + "input_tokens": 842416, + "output_tokens": 1323, + "total_tokens": 843739, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 265.54, + "avg_input_tokens": 495968.2, + "avg_output_tokens": 1198.2, + "avg_total_tokens": 497166.5, + "avg_turn_count": 12.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 42.87510895729065, + "agent_execution_time": 42.87510895729065, + "token_usage": { + "input_tokens": 271708, + "output_tokens": 2943, + "total_tokens": 274651, + "reasoning_tokens": 2474 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 72.09514212608337, + "agent_execution_time": 72.09514212608337, + "token_usage": { + "input_tokens": 490081, + "output_tokens": 4020, + "total_tokens": 494101, + "reasoning_tokens": 3048 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 61.30683755874634, + "agent_execution_time": 61.30683755874634, + "token_usage": { + "input_tokens": 291238, + "output_tokens": 4838, + "total_tokens": 296076, + "reasoning_tokens": 4048 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 32.9167058467865, + "agent_execution_time": 32.9167058467865, + "token_usage": { + "input_tokens": 152994, + "output_tokens": 1831, + "total_tokens": 154825, + "reasoning_tokens": 1431 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 52.3, + "avg_input_tokens": 301505.2, + "avg_output_tokens": 3408.0, + "avg_total_tokens": 304913.2, + "avg_turn_count": 8.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping_admin__customer_segmentation_setup.json b/task_results/shopping_admin__customer_segmentation_setup.json index 0c690b19..85ca7654 100644 --- a/task_results/shopping_admin__customer_segmentation_setup.json +++ b/task_results/shopping_admin__customer_segmentation_setup.json @@ -2,6 +2,73 @@ "task": "shopping_admin__customer_segmentation_setup", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 188.39300274848938, + "agent_execution_time": 188.39300274848938, + "token_usage": { + "input_tokens": 512646, + "output_tokens": 1522, + "total_tokens": 514168, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": false, + "execution_time": 50.723018407821655, + "agent_execution_time": 50.723018407821655, + "token_usage": { + "input_tokens": 4499, + "output_tokens": 1253, + "total_tokens": 5752, + "reasoning_tokens": 1209 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": true, + "execution_time": 772.9591863155365, + "agent_execution_time": 772.9591863155365, + "token_usage": { + "input_tokens": 366312, + "output_tokens": 14087, + "total_tokens": 380399, + "reasoning_tokens": 13250 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": true, + "execution_time": 417.7184669971466, + "agent_execution_time": 417.7184669971466, + "token_usage": { + "input_tokens": 191829, + "output_tokens": 10434, + "total_tokens": 202263, + "reasoning_tokens": 9824 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 357.45, + "avg_input_tokens": 268821.5, + "avg_output_tokens": 6824.0, + "avg_total_tokens": 275645.5, + "avg_turn_count": 10.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 301.27563405036926, - "agent_execution_time": 301.27563405036926, - "token_usage": { - "input_tokens": 641339, - "output_tokens": 2500, - "total_tokens": 643839, - "reasoning_tokens": null - }, - "turn_count": 20 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 301.28, - "avg_input_tokens": 641339.0, - "avg_output_tokens": 2500.0, - "avg_total_tokens": 643839.0, - "avg_turn_count": 20.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 124.42920970916748, + "agent_execution_time": 124.42920970916748, + "token_usage": { + "input_tokens": 662531, + "output_tokens": 2375, + "total_tokens": 664906, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": true, + "execution_time": 156.69361281394958, + "agent_execution_time": 156.69361281394958, + "token_usage": { + "input_tokens": 650714, + "output_tokens": 2664, + "total_tokens": 653378, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 129.42514824867249, + "agent_execution_time": 129.42514824867249, + "token_usage": { + "input_tokens": 461367, + "output_tokens": 2517, + "total_tokens": 463884, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": true, + "execution_time": 138.74757146835327, + "agent_execution_time": 138.74757146835327, + "token_usage": { + "input_tokens": 663494, + "output_tokens": 2452, + "total_tokens": 665946, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 137.32, + "avg_input_tokens": 609526.5, + "avg_output_tokens": 2502.0, + "avg_total_tokens": 612028.5, + "avg_turn_count": 19.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 331.5776889324188, + "agent_execution_time": 331.5776889324188, + "token_usage": { + "input_tokens": 672410, + "output_tokens": 3064, + "total_tokens": 675474, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": true, + "execution_time": 171.1226522922516, + "agent_execution_time": 171.1226522922516, + "token_usage": { + "input_tokens": 413752, + "output_tokens": 2902, + "total_tokens": 416654, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": true, + "execution_time": 167.30318188667297, + "agent_execution_time": 167.30318188667297, + "token_usage": { + "input_tokens": 586153, + "output_tokens": 2781, + "total_tokens": 588934, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": true, + "execution_time": 213.3192458152771, + "agent_execution_time": 213.3192458152771, + "token_usage": { + "input_tokens": 594703, + "output_tokens": 2829, + "total_tokens": 597532, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 220.83, + "avg_input_tokens": 566754.5, + "avg_output_tokens": 2894.0, + "avg_total_tokens": 569648.5, + "avg_turn_count": 17.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 214.1471290588379, + "agent_execution_time": 214.1471290588379, + "token_usage": { + "input_tokens": 643699, + "output_tokens": 1428, + "total_tokens": 645127, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": true, + "execution_time": 216.8935251235962, + "agent_execution_time": 216.8935251235962, + "token_usage": { + "input_tokens": 696510, + "output_tokens": 1270, + "total_tokens": 697780, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": true, + "execution_time": 441.1524147987366, + "agent_execution_time": 441.1524147987366, + "token_usage": { + "input_tokens": 722523, + "output_tokens": 1569, + "total_tokens": 724092, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": true, + "execution_time": 1328.1425185203552, + "agent_execution_time": 1328.1425185203552, + "token_usage": { + "input_tokens": 1033010, + "output_tokens": 1487, + "total_tokens": 1034497, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 550.08, + "avg_input_tokens": 773935.5, + "avg_output_tokens": 1438.5, + "avg_total_tokens": 775374.0, + "avg_turn_count": 21.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 61.54241704940796, + "agent_execution_time": 61.54241704940796, + "token_usage": { + "input_tokens": 418047, + "output_tokens": 2151, + "total_tokens": 420198, + "reasoning_tokens": 1467 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": true, + "execution_time": 63.27823305130005, + "agent_execution_time": 63.27823305130005, + "token_usage": { + "input_tokens": 473755, + "output_tokens": 1839, + "total_tokens": 475594, + "reasoning_tokens": 1036 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": true, + "execution_time": 56.19239258766174, + "agent_execution_time": 56.19239258766174, + "token_usage": { + "input_tokens": 342616, + "output_tokens": 2366, + "total_tokens": 344982, + "reasoning_tokens": 1610 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": true, + "execution_time": 52.29262828826904, + "agent_execution_time": 52.29262828826904, + "token_usage": { + "input_tokens": 385141, + "output_tokens": 2117, + "total_tokens": 387258, + "reasoning_tokens": 1370 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 58.33, + "avg_input_tokens": 404889.8, + "avg_output_tokens": 2118.2, + "avg_total_tokens": 407008.0, + "avg_turn_count": 14.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping_admin__fitness_promotion_strategy.json b/task_results/shopping_admin__fitness_promotion_strategy.json index 553322ed..ed4401ee 100644 --- a/task_results/shopping_admin__fitness_promotion_strategy.json +++ b/task_results/shopping_admin__fitness_promotion_strategy.json @@ -2,6 +2,73 @@ "task": "shopping_admin__fitness_promotion_strategy", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 386.5096573829651, + "agent_execution_time": 386.5096573829651, + "token_usage": { + "input_tokens": 286018, + "output_tokens": 1287, + "total_tokens": 287305, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 53.1364381313324, + "agent_execution_time": 53.1364381313324, + "token_usage": { + "input_tokens": 4693, + "output_tokens": 1418, + "total_tokens": 6111, + "reasoning_tokens": 1374 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": false, + "execution_time": 702.2816944122314, + "agent_execution_time": 702.2816944122314, + "token_usage": { + "input_tokens": 89318, + "output_tokens": 9545, + "total_tokens": 98863, + "reasoning_tokens": 9116 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 2079.1209819316864, + "agent_execution_time": 2079.1209819316864, + "token_usage": { + "input_tokens": 562407, + "output_tokens": 57413, + "total_tokens": 619820, + "reasoning_tokens": 56292 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 805.26, + "avg_input_tokens": 235609.0, + "avg_output_tokens": 17415.8, + "avg_total_tokens": 253024.8, + "avg_turn_count": 11.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 235.47408413887024, - "agent_execution_time": 235.47408413887024, - "token_usage": { - "input_tokens": 675712, - "output_tokens": 1525, - "total_tokens": 677237, - "reasoning_tokens": null - }, - "turn_count": 12 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 235.47, - "avg_input_tokens": 675712.0, - "avg_output_tokens": 1525.0, - "avg_total_tokens": 677237.0, - "avg_turn_count": 12.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 94.99269604682922, + "agent_execution_time": 94.99269604682922, + "token_usage": { + "input_tokens": 582810, + "output_tokens": 1486, + "total_tokens": 584296, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 112.86685919761658, + "agent_execution_time": 112.86685919761658, + "token_usage": { + "input_tokens": 581733, + "output_tokens": 1616, + "total_tokens": 583349, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 101.56730937957764, + "agent_execution_time": 101.56730937957764, + "token_usage": { + "input_tokens": 527583, + "output_tokens": 1325, + "total_tokens": 528908, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 91.0614743232727, + "agent_execution_time": 91.0614743232727, + "token_usage": { + "input_tokens": 575370, + "output_tokens": 1521, + "total_tokens": 576891, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 100.12, + "avg_input_tokens": 566874.0, + "avg_output_tokens": 1487.0, + "avg_total_tokens": 568361.0, + "avg_turn_count": 11.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 340.52450609207153, + "agent_execution_time": 340.52450609207153, + "token_usage": { + "input_tokens": 1426364, + "output_tokens": 2279, + "total_tokens": 1428643, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 173.18343925476074, + "agent_execution_time": 173.18343925476074, + "token_usage": { + "input_tokens": 736110, + "output_tokens": 2123, + "total_tokens": 738233, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 198.40538907051086, + "agent_execution_time": 198.40538907051086, + "token_usage": { + "input_tokens": 866853, + "output_tokens": 2291, + "total_tokens": 869144, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 194.69450616836548, + "agent_execution_time": 194.69450616836548, + "token_usage": { + "input_tokens": 221583, + "output_tokens": 1509, + "total_tokens": 223092, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 226.7, + "avg_input_tokens": 812727.5, + "avg_output_tokens": 2050.5, + "avg_total_tokens": 814778.0, + "avg_turn_count": 14.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 176.36079168319702, + "agent_execution_time": 176.36079168319702, + "token_usage": { + "input_tokens": 355450, + "output_tokens": 1175, + "total_tokens": 356625, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 167.2538239955902, + "agent_execution_time": 167.2538239955902, + "token_usage": { + "input_tokens": 236408, + "output_tokens": 1830, + "total_tokens": 238238, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 326.64486384391785, + "agent_execution_time": 326.64486384391785, + "token_usage": { + "input_tokens": 280645, + "output_tokens": 762, + "total_tokens": 281407, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 12.434375286102295, + "agent_execution_time": 12.434375286102295, + "token_usage": { + "input_tokens": 4693, + "output_tokens": 171, + "total_tokens": 4864, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 170.67, + "avg_input_tokens": 219299.0, + "avg_output_tokens": 984.5, + "avg_total_tokens": 220283.5, + "avg_turn_count": 10.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 122.45012879371643, + "agent_execution_time": 122.45012879371643, + "token_usage": { + "input_tokens": 1594248, + "output_tokens": 5811, + "total_tokens": 1600059, + "reasoning_tokens": 4760 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 104.87030553817749, + "agent_execution_time": 104.87030553817749, + "token_usage": { + "input_tokens": 2509994, + "output_tokens": 5751, + "total_tokens": 2515745, + "reasoning_tokens": 4916 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": false, + "execution_time": 135.1025629043579, + "agent_execution_time": 135.1025629043579, + "token_usage": { + "input_tokens": 4181174, + "output_tokens": 6681, + "total_tokens": 4187855, + "reasoning_tokens": 5525 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 394.6514859199524, + "agent_execution_time": 394.6514859199524, + "token_usage": { + "input_tokens": 10221640, + "output_tokens": 11088, + "total_tokens": 10232728, + "reasoning_tokens": 9332 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 189.27, + "avg_input_tokens": 4626764.0, + "avg_output_tokens": 7332.8, + "avg_total_tokens": 4634096.8, + "avg_turn_count": 18.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping_admin__marketing_customer_analysis.json b/task_results/shopping_admin__marketing_customer_analysis.json index 8cb0605e..c51e25fc 100644 --- a/task_results/shopping_admin__marketing_customer_analysis.json +++ b/task_results/shopping_admin__marketing_customer_analysis.json @@ -2,6 +2,73 @@ "task": "shopping_admin__marketing_customer_analysis", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 284.3517520427704, + "agent_execution_time": 284.3517520427704, + "token_usage": { + "input_tokens": 918065, + "output_tokens": 2388, + "total_tokens": 920453, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": false, + "execution_time": 43.57337212562561, + "agent_execution_time": 43.57337212562561, + "token_usage": { + "input_tokens": 4918, + "output_tokens": 1189, + "total_tokens": 6107, + "reasoning_tokens": 1148 + }, + "turn_count": 1 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1876.474007844925, + "agent_execution_time": 1876.474007844925, + "token_usage": { + "input_tokens": 615052, + "output_tokens": 39059, + "total_tokens": 654111, + "reasoning_tokens": 37768 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1377.8506574630737, + "agent_execution_time": 1377.8506574630737, + "token_usage": { + "input_tokens": 853288, + "output_tokens": 36326, + "total_tokens": 889614, + "reasoning_tokens": 34954 + }, + "turn_count": 21 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 895.56, + "avg_input_tokens": 597830.8, + "avg_output_tokens": 19740.5, + "avg_total_tokens": 617571.2, + "avg_turn_count": 15.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 522.9044888019562, - "agent_execution_time": 522.9044888019562, - "token_usage": { - "input_tokens": 1620249, - "output_tokens": 3647, - "total_tokens": 1623896, - "reasoning_tokens": null - }, - "turn_count": 28 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 522.9, - "avg_input_tokens": 1620249.0, - "avg_output_tokens": 3647.0, - "avg_total_tokens": 1623896.0, - "avg_turn_count": 28.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 193.00646543502808, + "agent_execution_time": 193.00646543502808, + "token_usage": { + "input_tokens": 1217047, + "output_tokens": 4250, + "total_tokens": 1221297, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": true, + "execution_time": 220.85941910743713, + "agent_execution_time": 220.85941910743713, + "token_usage": { + "input_tokens": 1530795, + "output_tokens": 4139, + "total_tokens": 1534934, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-3", + "success": true, + "execution_time": 191.45992827415466, + "agent_execution_time": 191.45992827415466, + "token_usage": { + "input_tokens": 1494771, + "output_tokens": 4180, + "total_tokens": 1498951, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-4", + "success": true, + "execution_time": 257.8118076324463, + "agent_execution_time": 257.8118076324463, + "token_usage": { + "input_tokens": 1537487, + "output_tokens": 4049, + "total_tokens": 1541536, + "reasoning_tokens": 0 + }, + "turn_count": 27 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 215.78, + "avg_input_tokens": 1445025.0, + "avg_output_tokens": 4154.5, + "avg_total_tokens": 1449179.5, + "avg_turn_count": 27.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 417.06060910224915, + "agent_execution_time": 417.06060910224915, + "token_usage": { + "input_tokens": 1124421, + "output_tokens": 4973, + "total_tokens": 1129394, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 216.70312690734863, + "agent_execution_time": 216.70312690734863, + "token_usage": { + "input_tokens": 861370, + "output_tokens": 4513, + "total_tokens": 865883, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 272.5494763851166, + "agent_execution_time": 272.5494763851166, + "token_usage": { + "input_tokens": 1161034, + "output_tokens": 5258, + "total_tokens": 1166292, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-4", + "success": false, + "execution_time": 325.47608065605164, + "agent_execution_time": 325.47608065605164, + "token_usage": { + "input_tokens": 518723, + "output_tokens": 2551, + "total_tokens": 521274, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 307.95, + "avg_input_tokens": 916387.0, + "avg_output_tokens": 4323.8, + "avg_total_tokens": 920710.8, + "avg_turn_count": 22.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 266.61186623573303, + "agent_execution_time": 266.61186623573303, + "token_usage": { + "input_tokens": 862730, + "output_tokens": 2359, + "total_tokens": 865089, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": true, + "execution_time": 260.6061067581177, + "agent_execution_time": 260.6061067581177, + "token_usage": { + "input_tokens": 712796, + "output_tokens": 2180, + "total_tokens": 714976, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 10.612432956695557, + "agent_execution_time": 10.612432956695557, + "token_usage": { + "input_tokens": 4918, + "output_tokens": 166, + "total_tokens": 5084, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 861.2593252658844, + "agent_execution_time": 861.2593252658844, + "token_usage": { + "input_tokens": 1228487, + "output_tokens": 1485, + "total_tokens": 1229972, + "reasoning_tokens": 0 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 349.77, + "avg_input_tokens": 702232.8, + "avg_output_tokens": 1547.5, + "avg_total_tokens": 703780.2, + "avg_turn_count": 17.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 80.241295337677, + "agent_execution_time": 80.241295337677, + "token_usage": { + "input_tokens": 586941, + "output_tokens": 3965, + "total_tokens": 590906, + "reasoning_tokens": 3002 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": true, + "execution_time": 81.10791397094727, + "agent_execution_time": 81.10791397094727, + "token_usage": { + "input_tokens": 708465, + "output_tokens": 3014, + "total_tokens": 711479, + "reasoning_tokens": 1995 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": true, + "execution_time": 87.85989713668823, + "agent_execution_time": 87.85989713668823, + "token_usage": { + "input_tokens": 712334, + "output_tokens": 3605, + "total_tokens": 715939, + "reasoning_tokens": 2365 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": true, + "execution_time": 91.39982771873474, + "agent_execution_time": 91.39982771873474, + "token_usage": { + "input_tokens": 759178, + "output_tokens": 3697, + "total_tokens": 762875, + "reasoning_tokens": 2621 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 85.15, + "avg_input_tokens": 691729.5, + "avg_output_tokens": 3570.2, + "avg_total_tokens": 695299.8, + "avg_turn_count": 17.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping_admin__ny_expansion_analysis.json b/task_results/shopping_admin__ny_expansion_analysis.json index 55d1ea37..e6aa30a9 100644 --- a/task_results/shopping_admin__ny_expansion_analysis.json +++ b/task_results/shopping_admin__ny_expansion_analysis.json @@ -2,6 +2,73 @@ "task": "shopping_admin__ny_expansion_analysis", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 462.2413377761841, + "agent_execution_time": 462.2413377761841, + "token_usage": { + "input_tokens": 737854, + "output_tokens": 1150, + "total_tokens": 739004, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 807.3879680633545, + "agent_execution_time": 807.3879680633545, + "token_usage": { + "input_tokens": 192102, + "output_tokens": 22091, + "total_tokens": 214193, + "reasoning_tokens": 21376 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1701.5678527355194, + "agent_execution_time": 1701.5678527355194, + "token_usage": { + "input_tokens": 424785, + "output_tokens": 37679, + "total_tokens": 462464, + "reasoning_tokens": 36752 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1100.2699790000916, + "agent_execution_time": 1100.2699790000916, + "token_usage": { + "input_tokens": 217291, + "output_tokens": 30839, + "total_tokens": 248130, + "reasoning_tokens": 30091 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1017.87, + "avg_input_tokens": 393008.0, + "avg_output_tokens": 22939.8, + "avg_total_tokens": 415947.8, + "avg_turn_count": 11.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 312.54434847831726, - "agent_execution_time": 312.54434847831726, - "token_usage": { - "input_tokens": 588846, - "output_tokens": 2206, - "total_tokens": 591052, - "reasoning_tokens": null - }, - "turn_count": 17 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 312.54, - "avg_input_tokens": 588846.0, - "avg_output_tokens": 2206.0, - "avg_total_tokens": 591052.0, - "avg_turn_count": 17.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 66.26638674736023, + "agent_execution_time": 66.26638674736023, + "token_usage": { + "input_tokens": 407541, + "output_tokens": 1122, + "total_tokens": 408663, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 70.09230470657349, + "agent_execution_time": 70.09230470657349, + "token_usage": { + "input_tokens": 278697, + "output_tokens": 1141, + "total_tokens": 279838, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 187.66044521331787, + "agent_execution_time": 187.66044521331787, + "token_usage": { + "input_tokens": 2324975, + "output_tokens": 3051, + "total_tokens": 2328026, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 221.8924160003662, + "agent_execution_time": 221.8924160003662, + "token_usage": { + "input_tokens": 2512099, + "output_tokens": 3918, + "total_tokens": 2516017, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 136.48, + "avg_input_tokens": 1380828.0, + "avg_output_tokens": 2308.0, + "avg_total_tokens": 1383136.0, + "avg_turn_count": 14.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 238.30118131637573, + "agent_execution_time": 238.30118131637573, + "token_usage": { + "input_tokens": 421551, + "output_tokens": 2833, + "total_tokens": 424384, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 117.67400908470154, + "agent_execution_time": 117.67400908470154, + "token_usage": { + "input_tokens": 448270, + "output_tokens": 2810, + "total_tokens": 451080, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 163.60476279258728, + "agent_execution_time": 163.60476279258728, + "token_usage": { + "input_tokens": 370831, + "output_tokens": 3040, + "total_tokens": 373871, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 18.722394466400146, + "agent_execution_time": 18.722394466400146, + "token_usage": { + "input_tokens": 10626, + "output_tokens": 122, + "total_tokens": 10748, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 134.58, + "avg_input_tokens": 312819.5, + "avg_output_tokens": 2201.2, + "avg_total_tokens": 315020.8, + "avg_turn_count": 12.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 257.78277111053467, + "agent_execution_time": 257.78277111053467, + "token_usage": { + "input_tokens": 913824, + "output_tokens": 2517, + "total_tokens": 916341, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 296.32229495048523, + "agent_execution_time": 296.32229495048523, + "token_usage": { + "input_tokens": 1125864, + "output_tokens": 2369, + "total_tokens": 1128233, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-3", + "success": false, + "execution_time": 472.8414161205292, + "agent_execution_time": 472.8414161205292, + "token_usage": { + "input_tokens": 767994, + "output_tokens": 1444, + "total_tokens": 769438, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-4", + "success": false, + "execution_time": 526.0267624855042, + "agent_execution_time": 526.0267624855042, + "token_usage": { + "input_tokens": 769993, + "output_tokens": 1197, + "total_tokens": 771190, + "reasoning_tokens": 0 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 388.24, + "avg_input_tokens": 894418.8, + "avg_output_tokens": 1881.8, + "avg_total_tokens": 896300.5, + "avg_turn_count": 23.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 63.6463520526886, + "agent_execution_time": 63.6463520526886, + "token_usage": { + "input_tokens": 324282, + "output_tokens": 3863, + "total_tokens": 328145, + "reasoning_tokens": 3178 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 58.401482343673706, + "agent_execution_time": 58.401482343673706, + "token_usage": { + "input_tokens": 366178, + "output_tokens": 3486, + "total_tokens": 369664, + "reasoning_tokens": 2761 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 92.82743620872498, + "agent_execution_time": 92.82743620872498, + "token_usage": { + "input_tokens": 1208374, + "output_tokens": 4410, + "total_tokens": 1212784, + "reasoning_tokens": 3431 + }, + "turn_count": 25 + }, + { + "run": "run-4", + "success": false, + "execution_time": 75.25112700462341, + "agent_execution_time": 75.25112700462341, + "token_usage": { + "input_tokens": 381780, + "output_tokens": 3687, + "total_tokens": 385467, + "reasoning_tokens": 2734 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 72.53, + "avg_input_tokens": 570153.5, + "avg_output_tokens": 3861.5, + "avg_total_tokens": 574015.0, + "avg_turn_count": 18.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping_admin__products_sales_analysis.json b/task_results/shopping_admin__products_sales_analysis.json index a44d3059..ce98002e 100644 --- a/task_results/shopping_admin__products_sales_analysis.json +++ b/task_results/shopping_admin__products_sales_analysis.json @@ -2,6 +2,73 @@ "task": "shopping_admin__products_sales_analysis", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 326.7992069721222, + "agent_execution_time": 326.7992069721222, + "token_usage": { + "input_tokens": 64845, + "output_tokens": 367, + "total_tokens": 65212, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 517.8571081161499, + "agent_execution_time": 517.8571081161499, + "token_usage": { + "input_tokens": 69644, + "output_tokens": 10748, + "total_tokens": 80392, + "reasoning_tokens": 10340 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 358.83156967163086, + "agent_execution_time": 358.83156967163086, + "token_usage": { + "input_tokens": 53575, + "output_tokens": 9362, + "total_tokens": 62937, + "reasoning_tokens": 9011 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 401.4616425037384, + "agent_execution_time": 401.4616425037384, + "token_usage": { + "input_tokens": 69527, + "output_tokens": 10315, + "total_tokens": 79842, + "reasoning_tokens": 9942 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 401.24, + "avg_input_tokens": 64397.8, + "avg_output_tokens": 7698.0, + "avg_total_tokens": 72095.8, + "avg_turn_count": 7.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 115.03723692893982, - "agent_execution_time": 115.03723692893982, - "token_usage": { - "input_tokens": 71720, - "output_tokens": 1140, - "total_tokens": 72860, - "reasoning_tokens": null - }, - "turn_count": 8 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 115.04, - "avg_input_tokens": 71720.0, - "avg_output_tokens": 1140.0, - "avg_total_tokens": 72860.0, - "avg_turn_count": 8.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 78.81223106384277, + "agent_execution_time": 78.81223106384277, + "token_usage": { + "input_tokens": 371201, + "output_tokens": 1252, + "total_tokens": 372453, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 79.45995903015137, + "agent_execution_time": 79.45995903015137, + "token_usage": { + "input_tokens": 367479, + "output_tokens": 1217, + "total_tokens": 368696, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 94.72774529457092, + "agent_execution_time": 94.72774529457092, + "token_usage": { + "input_tokens": 370438, + "output_tokens": 1256, + "total_tokens": 371694, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 73.70070576667786, + "agent_execution_time": 73.70070576667786, + "token_usage": { + "input_tokens": 369694, + "output_tokens": 1093, + "total_tokens": 370787, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 81.68, + "avg_input_tokens": 369703.0, + "avg_output_tokens": 1204.5, + "avg_total_tokens": 370907.5, + "avg_turn_count": 9.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 206.77240800857544, + "agent_execution_time": 206.77240800857544, + "token_usage": { + "input_tokens": 176306, + "output_tokens": 1450, + "total_tokens": 177756, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 81.09342336654663, + "agent_execution_time": 81.09342336654663, + "token_usage": { + "input_tokens": 176232, + "output_tokens": 1359, + "total_tokens": 177591, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 78.4146716594696, + "agent_execution_time": 78.4146716594696, + "token_usage": { + "input_tokens": 191188, + "output_tokens": 1326, + "total_tokens": 192514, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 24.652905702590942, + "agent_execution_time": 24.652905702590942, + "token_usage": { + "input_tokens": 2791, + "output_tokens": 0, + "total_tokens": 2791, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 97.73, + "avg_input_tokens": 136629.2, + "avg_output_tokens": 1033.8, + "avg_total_tokens": 137663.0, + "avg_turn_count": 5.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 81.78900957107544, + "agent_execution_time": 81.78900957107544, + "token_usage": { + "input_tokens": 60608, + "output_tokens": 424, + "total_tokens": 61032, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 67.3431282043457, + "agent_execution_time": 67.3431282043457, + "token_usage": { + "input_tokens": 60943, + "output_tokens": 455, + "total_tokens": 61398, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 215.31035828590393, + "agent_execution_time": 215.31035828590393, + "token_usage": { + "input_tokens": 69537, + "output_tokens": 474, + "total_tokens": 70011, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 12.990563869476318, + "agent_execution_time": 12.990563869476318, + "token_usage": { + "input_tokens": 4541, + "output_tokens": 295, + "total_tokens": 4836, + "reasoning_tokens": 0 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 94.36, + "avg_input_tokens": 48907.2, + "avg_output_tokens": 412.0, + "avg_total_tokens": 49319.2, + "avg_turn_count": 5.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 145.67451119422913, + "agent_execution_time": 145.67451119422913, + "token_usage": { + "input_tokens": 4776767, + "output_tokens": 5499, + "total_tokens": 4782266, + "reasoning_tokens": 4488 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 128.21493673324585, + "agent_execution_time": 128.21493673324585, + "token_usage": { + "input_tokens": 2196565, + "output_tokens": 5231, + "total_tokens": 2201796, + "reasoning_tokens": 4470 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 125.284499168396, + "agent_execution_time": 125.284499168396, + "token_usage": { + "input_tokens": 3493634, + "output_tokens": 5669, + "total_tokens": 3499303, + "reasoning_tokens": 4902 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": false, + "execution_time": 83.83782696723938, + "agent_execution_time": 83.83782696723938, + "token_usage": { + "input_tokens": 887609, + "output_tokens": 3681, + "total_tokens": 891290, + "reasoning_tokens": 3335 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 120.75, + "avg_input_tokens": 2838643.8, + "avg_output_tokens": 5020.0, + "avg_total_tokens": 2843663.8, + "avg_turn_count": 12.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping_admin__sales_inventory_analysis.json b/task_results/shopping_admin__sales_inventory_analysis.json index cec4bc0b..30638247 100644 --- a/task_results/shopping_admin__sales_inventory_analysis.json +++ b/task_results/shopping_admin__sales_inventory_analysis.json @@ -2,6 +2,73 @@ "task": "shopping_admin__sales_inventory_analysis", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 362.4113190174103, + "agent_execution_time": 362.4113190174103, + "token_usage": { + "input_tokens": 126111, + "output_tokens": 933, + "total_tokens": 127044, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 640.8920018672943, + "agent_execution_time": 640.8920018672943, + "token_usage": { + "input_tokens": 82988, + "output_tokens": 12424, + "total_tokens": 95412, + "reasoning_tokens": 12000 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 826.518173456192, + "agent_execution_time": 826.518173456192, + "token_usage": { + "input_tokens": 99299, + "output_tokens": 10965, + "total_tokens": 110264, + "reasoning_tokens": 10503 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 418.84778237342834, + "agent_execution_time": 418.84778237342834, + "token_usage": { + "input_tokens": 4690, + "output_tokens": 12154, + "total_tokens": 16844, + "reasoning_tokens": 8058 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 562.17, + "avg_input_tokens": 78272.0, + "avg_output_tokens": 9119.0, + "avg_total_tokens": 87391.0, + "avg_turn_count": 6.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 166.58150815963745, - "agent_execution_time": 166.58150815963745, - "token_usage": { - "input_tokens": 426336, - "output_tokens": 1046, - "total_tokens": 427382, - "reasoning_tokens": null - }, - "turn_count": 9 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 166.58, - "avg_input_tokens": 426336.0, - "avg_output_tokens": 1046.0, - "avg_total_tokens": 427382.0, - "avg_turn_count": 9.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 79.55335783958435, + "agent_execution_time": 79.55335783958435, + "token_usage": { + "input_tokens": 541427, + "output_tokens": 1256, + "total_tokens": 542683, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 85.35966205596924, + "agent_execution_time": 85.35966205596924, + "token_usage": { + "input_tokens": 538155, + "output_tokens": 1222, + "total_tokens": 539377, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 77.52313089370728, + "agent_execution_time": 77.52313089370728, + "token_usage": { + "input_tokens": 540754, + "output_tokens": 1172, + "total_tokens": 541926, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 87.099769115448, + "agent_execution_time": 87.099769115448, + "token_usage": { + "input_tokens": 544108, + "output_tokens": 1320, + "total_tokens": 545428, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 82.38, + "avg_input_tokens": 541111.0, + "avg_output_tokens": 1242.5, + "avg_total_tokens": 542353.5, + "avg_turn_count": 10.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 210.5877914428711, + "agent_execution_time": 210.5877914428711, + "token_usage": { + "input_tokens": 568493, + "output_tokens": 1802, + "total_tokens": 570295, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 109.1310703754425, + "agent_execution_time": 109.1310703754425, + "token_usage": { + "input_tokens": 221445, + "output_tokens": 2066, + "total_tokens": 223511, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 315.78510427474976, + "agent_execution_time": 315.78510427474976, + "token_usage": { + "input_tokens": 1092282, + "output_tokens": 3167, + "total_tokens": 1095449, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 78.96198010444641, + "agent_execution_time": 78.96198010444641, + "token_usage": { + "input_tokens": 11213, + "output_tokens": 282, + "total_tokens": 11495, + "reasoning_tokens": 0 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 178.62, + "avg_input_tokens": 473358.2, + "avg_output_tokens": 1829.2, + "avg_total_tokens": 475187.5, + "avg_turn_count": 9.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 110.81503748893738, + "agent_execution_time": 110.81503748893738, + "token_usage": { + "input_tokens": 123514, + "output_tokens": 1171, + "total_tokens": 124685, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 84.94838762283325, + "agent_execution_time": 84.94838762283325, + "token_usage": { + "input_tokens": 89734, + "output_tokens": 616, + "total_tokens": 90350, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 9.45291018486023, + "agent_execution_time": 9.45291018486023, + "token_usage": { + "input_tokens": 4690, + "output_tokens": 140, + "total_tokens": 4830, + "reasoning_tokens": 0 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": false, + "execution_time": 383.4228253364563, + "agent_execution_time": 383.4228253364563, + "token_usage": { + "input_tokens": 38465, + "output_tokens": 3474, + "total_tokens": 41939, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 147.16, + "avg_input_tokens": 64100.8, + "avg_output_tokens": 1350.2, + "avg_total_tokens": 65451.0, + "avg_turn_count": 5.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 204.37532448768616, + "agent_execution_time": 204.37532448768616, + "token_usage": { + "input_tokens": 7085073, + "output_tokens": 9530, + "total_tokens": 7094603, + "reasoning_tokens": 8545 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 219.33916592597961, + "agent_execution_time": 219.33916592597961, + "token_usage": { + "input_tokens": 7638200, + "output_tokens": 8739, + "total_tokens": 7646939, + "reasoning_tokens": 7610 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 95.73089456558228, + "agent_execution_time": 95.73089456558228, + "token_usage": { + "input_tokens": 2470849, + "output_tokens": 3356, + "total_tokens": 2474205, + "reasoning_tokens": 2697 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 138.0698573589325, + "agent_execution_time": 138.0698573589325, + "token_usage": { + "input_tokens": 2305589, + "output_tokens": 5552, + "total_tokens": 2311141, + "reasoning_tokens": 4664 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 164.38, + "avg_input_tokens": 4874927.8, + "avg_output_tokens": 6794.2, + "avg_total_tokens": 4881722.0, + "avg_turn_count": 15.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/shopping_admin__search_filtering_operations.json b/task_results/shopping_admin__search_filtering_operations.json index 3678ea04..2416a2e0 100644 --- a/task_results/shopping_admin__search_filtering_operations.json +++ b/task_results/shopping_admin__search_filtering_operations.json @@ -2,6 +2,73 @@ "task": "shopping_admin__search_filtering_operations", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 542.2417359352112, + "agent_execution_time": 542.2417359352112, + "token_usage": { + "input_tokens": 2053652, + "output_tokens": 6404, + "total_tokens": 2060056, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-2", + "success": false, + "execution_time": 737.8025443553925, + "agent_execution_time": 737.8025443553925, + "token_usage": { + "input_tokens": 120984, + "output_tokens": 19588, + "total_tokens": 140572, + "reasoning_tokens": 19109 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 230.39245128631592, + "agent_execution_time": 230.39245128631592, + "token_usage": { + "input_tokens": 40188, + "output_tokens": 4917, + "total_tokens": 45105, + "reasoning_tokens": 4661 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": true, + "execution_time": 2353.357953310013, + "agent_execution_time": 2353.357953310013, + "token_usage": { + "input_tokens": 706343, + "output_tokens": 64813, + "total_tokens": 771156, + "reasoning_tokens": 63258 + }, + "turn_count": 24 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 965.95, + "avg_input_tokens": 730291.8, + "avg_output_tokens": 23930.5, + "avg_total_tokens": 754222.2, + "avg_turn_count": 18.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 633.6722922325134, - "agent_execution_time": 633.6722922325134, - "token_usage": { - "input_tokens": 1615670, - "output_tokens": 4381, - "total_tokens": 1620051, - "reasoning_tokens": null - }, - "turn_count": 40 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 633.67, - "avg_input_tokens": 1615670.0, - "avg_output_tokens": 4381.0, - "avg_total_tokens": 1620051.0, - "avg_turn_count": 40.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 240.16096234321594, + "agent_execution_time": 240.16096234321594, + "token_usage": { + "input_tokens": 1390423, + "output_tokens": 5198, + "total_tokens": 1395621, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-2", + "success": true, + "execution_time": 218.5385138988495, + "agent_execution_time": 218.5385138988495, + "token_usage": { + "input_tokens": 1081965, + "output_tokens": 5329, + "total_tokens": 1087294, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-3", + "success": false, + "execution_time": 254.2272596359253, + "agent_execution_time": 254.2272596359253, + "token_usage": { + "input_tokens": 1481207, + "output_tokens": 5645, + "total_tokens": 1486852, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + { + "run": "run-4", + "success": false, + "execution_time": 262.48115944862366, + "agent_execution_time": 262.48115944862366, + "token_usage": { + "input_tokens": 1563942, + "output_tokens": 4976, + "total_tokens": 1568918, + "reasoning_tokens": 0 + }, + "turn_count": 36 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 243.85, + "avg_input_tokens": 1379384.2, + "avg_output_tokens": 5287.0, + "avg_total_tokens": 1384671.2, + "avg_turn_count": 32.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 498.4454791545868, + "agent_execution_time": 498.4454791545868, + "token_usage": { + "input_tokens": 1338290, + "output_tokens": 6084, + "total_tokens": 1344374, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-2", + "success": false, + "execution_time": 249.90884447097778, + "agent_execution_time": 249.90884447097778, + "token_usage": { + "input_tokens": 1010859, + "output_tokens": 6047, + "total_tokens": 1016906, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-3", + "success": true, + "execution_time": 309.37129759788513, + "agent_execution_time": 309.37129759788513, + "token_usage": { + "input_tokens": 1228578, + "output_tokens": 5963, + "total_tokens": 1234541, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-4", + "success": false, + "execution_time": 7.914529085159302, + "agent_execution_time": 7.914529085159302, + "token_usage": { + "input_tokens": 6947, + "output_tokens": 170, + "total_tokens": 7117, + "reasoning_tokens": 0 + }, + "turn_count": 2 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 266.41, + "avg_input_tokens": 896168.5, + "avg_output_tokens": 4566.0, + "avg_total_tokens": 900734.5, + "avg_turn_count": 24.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 419.3902413845062, + "agent_execution_time": 419.3902413845062, + "token_usage": { + "input_tokens": 1609071, + "output_tokens": 4041, + "total_tokens": 1613112, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + { + "run": "run-2", + "success": false, + "execution_time": 422.50532579421997, + "agent_execution_time": 422.50532579421997, + "token_usage": { + "input_tokens": 1733315, + "output_tokens": 3994, + "total_tokens": 1737309, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + { + "run": "run-3", + "success": true, + "execution_time": 348.3605172634125, + "agent_execution_time": 348.3605172634125, + "token_usage": { + "input_tokens": 299768, + "output_tokens": 5065, + "total_tokens": 304833, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 547.2540693283081, + "agent_execution_time": 547.2540693283081, + "token_usage": { + "input_tokens": 558959, + "output_tokens": 5737, + "total_tokens": 564696, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 434.38, + "avg_input_tokens": 1050278.2, + "avg_output_tokens": 4709.2, + "avg_total_tokens": 1054987.5, + "avg_turn_count": 23.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 92.69794154167175, + "agent_execution_time": 92.69794154167175, + "token_usage": { + "input_tokens": 617412, + "output_tokens": 4644, + "total_tokens": 622056, + "reasoning_tokens": 3331 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 137.17717957496643, + "agent_execution_time": 137.17717957496643, + "token_usage": { + "input_tokens": 1260826, + "output_tokens": 6168, + "total_tokens": 1266994, + "reasoning_tokens": 4211 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 127.40874028205872, + "agent_execution_time": 127.40874028205872, + "token_usage": { + "input_tokens": 872212, + "output_tokens": 5947, + "total_tokens": 878159, + "reasoning_tokens": 4279 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": true, + "execution_time": 105.42771601676941, + "agent_execution_time": 105.42771601676941, + "token_usage": { + "input_tokens": 587191, + "output_tokens": 5029, + "total_tokens": 592220, + "reasoning_tokens": 3644 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 115.68, + "avg_input_tokens": 834410.2, + "avg_output_tokens": 5447.0, + "avg_total_tokens": 839857.2, + "avg_turn_count": 19.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/sports__baseball_player_analysis.json b/task_results/sports__baseball_player_analysis.json index 53d8d765..567f834a 100644 --- a/task_results/sports__baseball_player_analysis.json +++ b/task_results/sports__baseball_player_analysis.json @@ -2,6 +2,73 @@ "task": "sports__baseball_player_analysis", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 501.9244418144226, + "agent_execution_time": 501.9244418144226, + "token_usage": { + "input_tokens": 928853, + "output_tokens": 7284, + "total_tokens": 936137, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + { + "run": "run-2", + "success": false, + "execution_time": 482.90266966819763, + "agent_execution_time": 482.90266966819763, + "token_usage": { + "input_tokens": 986703, + "output_tokens": 6934, + "total_tokens": 993637, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + { + "run": "run-3", + "success": false, + "execution_time": 98.48183274269104, + "agent_execution_time": 98.48183274269104, + "token_usage": { + "input_tokens": 34696, + "output_tokens": 2212, + "total_tokens": 36908, + "reasoning_tokens": 2019 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 61.051225900650024, + "agent_execution_time": 61.051225900650024, + "token_usage": { + "input_tokens": 2759, + "output_tokens": 1633, + "total_tokens": 4392, + "reasoning_tokens": 1601 + }, + "turn_count": 1 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 286.09, + "avg_input_tokens": 488252.8, + "avg_output_tokens": 4515.8, + "avg_total_tokens": 492768.5, + "avg_turn_count": 21.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 444.3078725337982, - "agent_execution_time": 444.3078725337982, - "token_usage": { - "input_tokens": 158872, - "output_tokens": 9194, - "total_tokens": 168066, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 444.31, - "avg_input_tokens": 158872.0, - "avg_output_tokens": 9194.0, - "avg_total_tokens": 168066.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 111.87357664108276, + "agent_execution_time": 111.87357664108276, + "token_usage": { + "input_tokens": 441708, + "output_tokens": 5164, + "total_tokens": 446872, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 147.65282464027405, + "agent_execution_time": 147.65282464027405, + "token_usage": { + "input_tokens": 690167, + "output_tokens": 6269, + "total_tokens": 696436, + "reasoning_tokens": 0 + }, + "turn_count": 28 + }, + { + "run": "run-3", + "success": false, + "execution_time": 122.88530802726746, + "agent_execution_time": 122.88530802726746, + "token_usage": { + "input_tokens": 435252, + "output_tokens": 5296, + "total_tokens": 440548, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 132.57741951942444, + "agent_execution_time": 132.57741951942444, + "token_usage": { + "input_tokens": 551234, + "output_tokens": 6096, + "total_tokens": 557330, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 128.75, + "avg_input_tokens": 529590.2, + "avg_output_tokens": 5706.2, + "avg_total_tokens": 535296.5, + "avg_turn_count": 22.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 384.7446782588959, + "agent_execution_time": 384.7446782588959, + "token_usage": { + "input_tokens": 835216, + "output_tokens": 9027, + "total_tokens": 844243, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + { + "run": "run-2", + "success": true, + "execution_time": 640.1871137619019, + "agent_execution_time": 640.1871137619019, + "token_usage": { + "input_tokens": 701876, + "output_tokens": 8639, + "total_tokens": 710515, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-3", + "success": false, + "execution_time": 318.6987292766571, + "agent_execution_time": 318.6987292766571, + "token_usage": { + "input_tokens": 922846, + "output_tokens": 7878, + "total_tokens": 930724, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + { + "run": "run-4", + "success": false, + "execution_time": 373.0751893520355, + "agent_execution_time": 373.0751893520355, + "token_usage": { + "input_tokens": 1093749, + "output_tokens": 10374, + "total_tokens": 1104123, + "reasoning_tokens": 0 + }, + "turn_count": 44 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 429.18, + "avg_input_tokens": 888421.8, + "avg_output_tokens": 8979.5, + "avg_total_tokens": 897401.2, + "avg_turn_count": 40.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 542.8017468452454, + "agent_execution_time": 542.8017468452454, + "token_usage": { + "input_tokens": 1109408, + "output_tokens": 8989, + "total_tokens": 1118397, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + { + "run": "run-2", + "success": true, + "execution_time": 581.1975631713867, + "agent_execution_time": 581.1975631713867, + "token_usage": { + "input_tokens": 964152, + "output_tokens": 8318, + "total_tokens": 972470, + "reasoning_tokens": 0 + }, + "turn_count": 46 + }, + { + "run": "run-3", + "success": false, + "execution_time": 453.684809923172, + "agent_execution_time": 453.684809923172, + "token_usage": { + "input_tokens": 1024908, + "output_tokens": 5830, + "total_tokens": 1030738, + "reasoning_tokens": 0 + }, + "turn_count": 41 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.33480954170227, + "agent_execution_time": 8.33480954170227, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 396.5, + "avg_input_tokens": 774617.0, + "avg_output_tokens": 5784.2, + "avg_total_tokens": 780401.2, + "avg_turn_count": 32.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 106.88859868049622, + "agent_execution_time": 106.88859868049622, + "token_usage": { + "input_tokens": 807104, + "output_tokens": 8638, + "total_tokens": 815742, + "reasoning_tokens": 5179 + }, + "turn_count": 42 + }, + { + "run": "run-2", + "success": false, + "execution_time": 129.7336893081665, + "agent_execution_time": 129.7336893081665, + "token_usage": { + "input_tokens": 1047254, + "output_tokens": 9371, + "total_tokens": 1056625, + "reasoning_tokens": 6535 + }, + "turn_count": 47 + }, + { + "run": "run-3", + "success": false, + "execution_time": 98.44750738143921, + "agent_execution_time": 98.44750738143921, + "token_usage": { + "input_tokens": 788054, + "output_tokens": 8278, + "total_tokens": 796332, + "reasoning_tokens": 5188 + }, + "turn_count": 39 + }, + { + "run": "run-4", + "success": false, + "execution_time": 92.44631671905518, + "agent_execution_time": 92.44631671905518, + "token_usage": { + "input_tokens": 644706, + "output_tokens": 8405, + "total_tokens": 653111, + "reasoning_tokens": 5794 + }, + "turn_count": 34 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 106.88, + "avg_input_tokens": 821779.5, + "avg_output_tokens": 8673.0, + "avg_total_tokens": 830452.5, + "avg_turn_count": 40.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/sports__participant_report_optimization.json b/task_results/sports__participant_report_optimization.json index 883fabe0..eb63fac1 100644 --- a/task_results/sports__participant_report_optimization.json +++ b/task_results/sports__participant_report_optimization.json @@ -2,6 +2,73 @@ "task": "sports__participant_report_optimization", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 185.82887816429138, + "agent_execution_time": 185.82887816429138, + "token_usage": { + "input_tokens": 192305, + "output_tokens": 2527, + "total_tokens": 194832, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-2", + "success": true, + "execution_time": 226.8004548549652, + "agent_execution_time": 226.8004548549652, + "token_usage": { + "input_tokens": 284288, + "output_tokens": 3460, + "total_tokens": 287748, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": true, + "execution_time": 628.985714673996, + "agent_execution_time": 628.985714673996, + "token_usage": { + "input_tokens": 123472, + "output_tokens": 15916, + "total_tokens": 139388, + "reasoning_tokens": 13929 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": true, + "execution_time": 438.4819691181183, + "agent_execution_time": 438.4819691181183, + "token_usage": { + "input_tokens": 24074, + "output_tokens": 11072, + "total_tokens": 35146, + "reasoning_tokens": 9607 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 370.02, + "avg_input_tokens": 156034.8, + "avg_output_tokens": 8243.8, + "avg_total_tokens": 164278.5, + "avg_turn_count": 13.25, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 277.8676087856293, - "agent_execution_time": 277.8676087856293, - "token_usage": { - "input_tokens": 123864, - "output_tokens": 4546, - "total_tokens": 128410, - "reasoning_tokens": null - }, - "turn_count": 22 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 277.87, - "avg_input_tokens": 123864.0, - "avg_output_tokens": 4546.0, - "avg_total_tokens": 128410.0, - "avg_turn_count": 22.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 84.42110967636108, + "agent_execution_time": 84.42110967636108, + "token_usage": { + "input_tokens": 188109, + "output_tokens": 3587, + "total_tokens": 191696, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": true, + "execution_time": 84.15524506568909, + "agent_execution_time": 84.15524506568909, + "token_usage": { + "input_tokens": 211772, + "output_tokens": 3685, + "total_tokens": 215457, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": true, + "execution_time": 114.46356654167175, + "agent_execution_time": 114.46356654167175, + "token_usage": { + "input_tokens": 288767, + "output_tokens": 5037, + "total_tokens": 293804, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": true, + "execution_time": 84.08264136314392, + "agent_execution_time": 84.08264136314392, + "token_usage": { + "input_tokens": 177526, + "output_tokens": 3980, + "total_tokens": 181506, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 91.78, + "avg_input_tokens": 216543.5, + "avg_output_tokens": 4072.2, + "avg_total_tokens": 220615.8, + "avg_turn_count": 14.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 174.52094841003418, + "agent_execution_time": 174.52094841003418, + "token_usage": { + "input_tokens": 183212, + "output_tokens": 3138, + "total_tokens": 186350, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": true, + "execution_time": 210.1871988773346, + "agent_execution_time": 210.1871988773346, + "token_usage": { + "input_tokens": 219995, + "output_tokens": 2557, + "total_tokens": 222552, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": true, + "execution_time": 158.8601644039154, + "agent_execution_time": 158.8601644039154, + "token_usage": { + "input_tokens": 260501, + "output_tokens": 4276, + "total_tokens": 264777, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": true, + "execution_time": 193.19554924964905, + "agent_execution_time": 193.19554924964905, + "token_usage": { + "input_tokens": 256325, + "output_tokens": 4033, + "total_tokens": 260358, + "reasoning_tokens": 0 + }, + "turn_count": 18 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 184.19, + "avg_input_tokens": 230008.2, + "avg_output_tokens": 3501.0, + "avg_total_tokens": 233509.2, + "avg_turn_count": 17.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 185.02157998085022, + "agent_execution_time": 185.02157998085022, + "token_usage": { + "input_tokens": 206543, + "output_tokens": 2498, + "total_tokens": 209041, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": true, + "execution_time": 198.7837188243866, + "agent_execution_time": 198.7837188243866, + "token_usage": { + "input_tokens": 218552, + "output_tokens": 2606, + "total_tokens": 221158, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": true, + "execution_time": 190.93105626106262, + "agent_execution_time": 190.93105626106262, + "token_usage": { + "input_tokens": 214355, + "output_tokens": 2742, + "total_tokens": 217097, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.22848629951477, + "agent_execution_time": 8.22848629951477, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 145.74, + "avg_input_tokens": 159862.5, + "avg_output_tokens": 1961.5, + "avg_total_tokens": 161824.0, + "avg_turn_count": 13.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 58.26852893829346, + "agent_execution_time": 58.26852893829346, + "token_usage": { + "input_tokens": 101091, + "output_tokens": 6738, + "total_tokens": 107829, + "reasoning_tokens": 4290 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": true, + "execution_time": 64.63362216949463, + "agent_execution_time": 64.63362216949463, + "token_usage": { + "input_tokens": 141053, + "output_tokens": 5087, + "total_tokens": 146140, + "reasoning_tokens": 2796 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": true, + "execution_time": 60.86538004875183, + "agent_execution_time": 60.86538004875183, + "token_usage": { + "input_tokens": 161705, + "output_tokens": 7298, + "total_tokens": 169003, + "reasoning_tokens": 4612 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": true, + "execution_time": 30.700495719909668, + "agent_execution_time": 30.700495719909668, + "token_usage": { + "input_tokens": 142979, + "output_tokens": 3425, + "total_tokens": 146404, + "reasoning_tokens": 2250 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 53.62, + "avg_input_tokens": 136707.0, + "avg_output_tokens": 5637.0, + "avg_total_tokens": 142344.0, + "avg_turn_count": 13.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/sports__team_roster_management.json b/task_results/sports__team_roster_management.json index a09345fd..5dfb319a 100644 --- a/task_results/sports__team_roster_management.json +++ b/task_results/sports__team_roster_management.json @@ -2,6 +2,73 @@ "task": "sports__team_roster_management", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 385.0476920604706, + "agent_execution_time": 385.0476920604706, + "token_usage": { + "input_tokens": 919773, + "output_tokens": 4478, + "total_tokens": 924251, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + { + "run": "run-2", + "success": false, + "execution_time": 356.95028591156006, + "agent_execution_time": 356.95028591156006, + "token_usage": { + "input_tokens": 605303, + "output_tokens": 4568, + "total_tokens": 609871, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + { + "run": "run-3", + "success": false, + "execution_time": 677.6811947822571, + "agent_execution_time": 677.6811947822571, + "token_usage": { + "input_tokens": 111057, + "output_tokens": 18782, + "total_tokens": 129839, + "reasoning_tokens": 17590 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1398.0381712913513, + "agent_execution_time": 1398.0381712913513, + "token_usage": { + "input_tokens": 126813, + "output_tokens": 32319, + "total_tokens": 159132, + "reasoning_tokens": 28703 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 704.43, + "avg_input_tokens": 440736.5, + "avg_output_tokens": 15036.8, + "avg_total_tokens": 455773.2, + "avg_turn_count": 26.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 325.1091368198395, - "agent_execution_time": 325.1091368198395, - "token_usage": { - "input_tokens": 158775, - "output_tokens": 5538, - "total_tokens": 164313, - "reasoning_tokens": null - }, - "turn_count": 25 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 325.11, - "avg_input_tokens": 158775.0, - "avg_output_tokens": 5538.0, - "avg_total_tokens": 164313.0, - "avg_turn_count": 25.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 124.91776180267334, + "agent_execution_time": 124.91776180267334, + "token_usage": { + "input_tokens": 450145, + "output_tokens": 4996, + "total_tokens": 455141, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 111.1750020980835, + "agent_execution_time": 111.1750020980835, + "token_usage": { + "input_tokens": 393854, + "output_tokens": 4447, + "total_tokens": 398301, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": false, + "execution_time": 135.88168215751648, + "agent_execution_time": 135.88168215751648, + "token_usage": { + "input_tokens": 432908, + "output_tokens": 5114, + "total_tokens": 438022, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": false, + "execution_time": 119.40657472610474, + "agent_execution_time": 119.40657472610474, + "token_usage": { + "input_tokens": 415448, + "output_tokens": 4817, + "total_tokens": 420265, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 122.85, + "avg_input_tokens": 423088.8, + "avg_output_tokens": 4843.5, + "avg_total_tokens": 427932.2, + "avg_turn_count": 23.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 427.552960395813, + "agent_execution_time": 427.552960395813, + "token_usage": { + "input_tokens": 486967, + "output_tokens": 5060, + "total_tokens": 492027, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-2", + "success": true, + "execution_time": 384.1921184062958, + "agent_execution_time": 384.1921184062958, + "token_usage": { + "input_tokens": 523628, + "output_tokens": 5379, + "total_tokens": 529007, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-3", + "success": true, + "execution_time": 196.74167203903198, + "agent_execution_time": 196.74167203903198, + "token_usage": { + "input_tokens": 581351, + "output_tokens": 5199, + "total_tokens": 586550, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-4", + "success": true, + "execution_time": 260.12403082847595, + "agent_execution_time": 260.12403082847595, + "token_usage": { + "input_tokens": 679800, + "output_tokens": 6210, + "total_tokens": 686010, + "reasoning_tokens": 0 + }, + "turn_count": 35 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 317.15, + "avg_input_tokens": 567936.5, + "avg_output_tokens": 5462.0, + "avg_total_tokens": 573398.5, + "avg_turn_count": 33.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 399.92182302474976, + "agent_execution_time": 399.92182302474976, + "token_usage": { + "input_tokens": 709903, + "output_tokens": 4718, + "total_tokens": 714621, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + { + "run": "run-2", + "success": false, + "execution_time": 418.0447723865509, + "agent_execution_time": 418.0447723865509, + "token_usage": { + "input_tokens": 851051, + "output_tokens": 4494, + "total_tokens": 855545, + "reasoning_tokens": 0 + }, + "turn_count": 46 + }, + { + "run": "run-3", + "success": false, + "execution_time": 279.40716457366943, + "agent_execution_time": 279.40716457366943, + "token_usage": { + "input_tokens": 542925, + "output_tokens": 3054, + "total_tokens": 545979, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.270027160644531, + "agent_execution_time": 8.270027160644531, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 276.41, + "avg_input_tokens": 525969.8, + "avg_output_tokens": 3066.5, + "avg_total_tokens": 529036.2, + "avg_turn_count": 29.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 57.16362690925598, + "agent_execution_time": 57.16362690925598, + "token_usage": { + "input_tokens": 254769, + "output_tokens": 6013, + "total_tokens": 260782, + "reasoning_tokens": 4531 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": true, + "execution_time": 125.15681409835815, + "agent_execution_time": 125.15681409835815, + "token_usage": { + "input_tokens": 738544, + "output_tokens": 9605, + "total_tokens": 748149, + "reasoning_tokens": 6000 + }, + "turn_count": 40 + }, + { + "run": "run-3", + "success": false, + "execution_time": 76.2571792602539, + "agent_execution_time": 76.2571792602539, + "token_usage": { + "input_tokens": 262372, + "output_tokens": 6881, + "total_tokens": 269253, + "reasoning_tokens": 5080 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 63.27751708030701, + "agent_execution_time": 63.27751708030701, + "token_usage": { + "input_tokens": 284138, + "output_tokens": 6667, + "total_tokens": 290805, + "reasoning_tokens": 5088 + }, + "turn_count": 22 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 80.46, + "avg_input_tokens": 384955.8, + "avg_output_tokens": 7291.5, + "avg_total_tokens": 392247.2, + "avg_turn_count": 25.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/standard_operating_procedure__deployment_process_sop.json b/task_results/standard_operating_procedure__deployment_process_sop.json index f19321d2..bba90007 100644 --- a/task_results/standard_operating_procedure__deployment_process_sop.json +++ b/task_results/standard_operating_procedure__deployment_process_sop.json @@ -2,6 +2,73 @@ "task": "standard_operating_procedure__deployment_process_sop", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 1545.4248478412628, + "agent_execution_time": 1545.4248478412628, + "token_usage": { + "input_tokens": 230617, + "output_tokens": 41104, + "total_tokens": 271721, + "reasoning_tokens": 35917 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 3600.03568315506, + "agent_execution_time": 3600.03568315506, + "token_usage": { + "input_tokens": 817385, + "output_tokens": 61242, + "total_tokens": 878627, + "reasoning_tokens": 55535 + }, + "turn_count": 22 + }, + { + "run": "run-3", + "success": true, + "execution_time": 518.9985489845276, + "agent_execution_time": 518.9985489845276, + "token_usage": { + "input_tokens": 2779599, + "output_tokens": 4445, + "total_tokens": 2784044, + "reasoning_tokens": 0 + }, + "turn_count": 49 + }, + { + "run": "run-4", + "success": true, + "execution_time": 1990.661957025528, + "agent_execution_time": 1990.661957025528, + "token_usage": { + "input_tokens": 365481, + "output_tokens": 45234, + "total_tokens": 410715, + "reasoning_tokens": 37998 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 1913.78, + "avg_input_tokens": 1048270.5, + "avg_output_tokens": 38006.2, + "avg_total_tokens": 1086276.8, + "avg_turn_count": 22.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 584.4173250198364, - "agent_execution_time": 584.4173250198364, - "token_usage": { - "input_tokens": 1108203, - "output_tokens": 5896, - "total_tokens": 1114099, - "reasoning_tokens": null - }, - "turn_count": 36 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 584.42, - "avg_input_tokens": 1108203.0, - "avg_output_tokens": 5896.0, - "avg_total_tokens": 1114099.0, - "avg_turn_count": 36.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 317.0396327972412, + "agent_execution_time": 317.0396327972412, + "token_usage": { + "input_tokens": 2450141, + "output_tokens": 8400, + "total_tokens": 2458541, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + { + "run": "run-2", + "success": true, + "execution_time": 198.16494750976562, + "agent_execution_time": 198.16494750976562, + "token_usage": { + "input_tokens": 646837, + "output_tokens": 5827, + "total_tokens": 652664, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-3", + "success": false, + "execution_time": 358.2421863079071, + "agent_execution_time": 358.2421863079071, + "token_usage": { + "input_tokens": 1396044, + "output_tokens": 11315, + "total_tokens": 1407359, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-4", + "success": false, + "execution_time": 292.14235186576843, + "agent_execution_time": 292.14235186576843, + "token_usage": { + "input_tokens": 1633368, + "output_tokens": 7705, + "total_tokens": 1641073, + "reasoning_tokens": 0 + }, + "turn_count": 27 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 291.4, + "avg_input_tokens": 1531597.5, + "avg_output_tokens": 8311.8, + "avg_total_tokens": 1539909.2, + "avg_turn_count": 27.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 477.6065456867218, + "agent_execution_time": 477.6065456867218, + "token_usage": { + "input_tokens": 2561464, + "output_tokens": 5155, + "total_tokens": 2566619, + "reasoning_tokens": 0 + }, + "turn_count": 59 + }, + { + "run": "run-2", + "success": false, + "execution_time": 31.954814195632935, + "agent_execution_time": 31.954814195632935, + "token_usage": { + "input_tokens": 141943, + "output_tokens": 568, + "total_tokens": 142511, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1501.3019998073578, + "agent_execution_time": 1501.3019998073578, + "token_usage": { + "input_tokens": 4945010, + "output_tokens": 11586, + "total_tokens": 4956596, + "reasoning_tokens": 0 + }, + "turn_count": 82 + }, + { + "run": "run-4", + "success": false, + "execution_time": 502.6164765357971, + "agent_execution_time": 502.6164765357971, + "token_usage": { + "input_tokens": 3365261, + "output_tokens": 6874, + "total_tokens": 3372135, + "reasoning_tokens": 0 + }, + "turn_count": 58 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 628.37, + "avg_input_tokens": 2753419.5, + "avg_output_tokens": 6045.8, + "avg_total_tokens": 2759465.2, + "avg_turn_count": 52.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 398.45121812820435, + "agent_execution_time": 398.45121812820435, + "token_usage": { + "input_tokens": 1193083, + "output_tokens": 3748, + "total_tokens": 1196831, + "reasoning_tokens": 0 + }, + "turn_count": 40 + }, + { + "run": "run-2", + "success": false, + "execution_time": 110.67174291610718, + "agent_execution_time": 110.67174291610718, + "token_usage": { + "input_tokens": 162119, + "output_tokens": 2083, + "total_tokens": 164202, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": true, + "execution_time": 469.4976861476898, + "agent_execution_time": 469.4976861476898, + "token_usage": { + "input_tokens": 2488535, + "output_tokens": 4148, + "total_tokens": 2492683, + "reasoning_tokens": 0 + }, + "turn_count": 44 + }, + { + "run": "run-4", + "success": false, + "execution_time": 438.0975124835968, + "agent_execution_time": 438.0975124835968, + "token_usage": { + "input_tokens": 1193063, + "output_tokens": 4136, + "total_tokens": 1197199, + "reasoning_tokens": 0 + }, + "turn_count": 41 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 354.18, + "avg_input_tokens": 1259200.0, + "avg_output_tokens": 3528.8, + "avg_total_tokens": 1262728.8, + "avg_turn_count": 33.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 85.01771593093872, + "agent_execution_time": 85.01771593093872, + "token_usage": { + "input_tokens": 166000, + "output_tokens": 8425, + "total_tokens": 174425, + "reasoning_tokens": 5554 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 61.74423956871033, + "agent_execution_time": 61.74423956871033, + "token_usage": { + "input_tokens": 231409, + "output_tokens": 7155, + "total_tokens": 238564, + "reasoning_tokens": 5885 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1787.9060208797455, + "agent_execution_time": 1787.9060208797455, + "token_usage": { + "input_tokens": 818201, + "output_tokens": 118904, + "total_tokens": 937105, + "reasoning_tokens": 4867 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 733.1941394805908, + "agent_execution_time": 733.1941394805908, + "token_usage": { + "input_tokens": 484239, + "output_tokens": 98344, + "total_tokens": 582583, + "reasoning_tokens": 6242 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 666.97, + "avg_input_tokens": 424962.2, + "avg_output_tokens": 58207.0, + "avg_total_tokens": 483169.2, + "avg_turn_count": 12.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/standard_operating_procedure__section_organization.json b/task_results/standard_operating_procedure__section_organization.json index 12a92399..4180ef05 100644 --- a/task_results/standard_operating_procedure__section_organization.json +++ b/task_results/standard_operating_procedure__section_organization.json @@ -2,6 +2,73 @@ "task": "standard_operating_procedure__section_organization", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 1947.6300837993622, + "agent_execution_time": 1947.6300837993622, + "token_usage": { + "input_tokens": 141731, + "output_tokens": 53367, + "total_tokens": 195098, + "reasoning_tokens": 51145 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 920.9546501636505, + "agent_execution_time": 920.9546501636505, + "token_usage": { + "input_tokens": 30058, + "output_tokens": 22077, + "total_tokens": 52135, + "reasoning_tokens": 21702 + }, + "turn_count": 3 + }, + { + "run": "run-3", + "success": false, + "execution_time": 465.8426823616028, + "agent_execution_time": 465.8426823616028, + "token_usage": { + "input_tokens": 947240, + "output_tokens": 6178, + "total_tokens": 953418, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + { + "run": "run-4", + "success": false, + "execution_time": 2128.761714696884, + "agent_execution_time": 2128.761714696884, + "token_usage": { + "input_tokens": 79161, + "output_tokens": 35572, + "total_tokens": 114733, + "reasoning_tokens": 33225 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1365.8, + "avg_input_tokens": 299547.5, + "avg_output_tokens": 29298.5, + "avg_total_tokens": 328846.0, + "avg_turn_count": 13.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 361.242742061615, - "agent_execution_time": 361.242742061615, - "token_usage": { - "input_tokens": 501656, - "output_tokens": 4149, - "total_tokens": 505805, - "reasoning_tokens": null - }, - "turn_count": 24 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 361.24, - "avg_input_tokens": 501656.0, - "avg_output_tokens": 4149.0, - "avg_total_tokens": 505805.0, - "avg_turn_count": 24.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 129.71568870544434, + "agent_execution_time": 129.71568870544434, + "token_usage": { + "input_tokens": 283715, + "output_tokens": 4008, + "total_tokens": 287723, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 237.5986340045929, + "agent_execution_time": 237.5986340045929, + "token_usage": { + "input_tokens": 625378, + "output_tokens": 7882, + "total_tokens": 633260, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 126.77526426315308, + "agent_execution_time": 126.77526426315308, + "token_usage": { + "input_tokens": 210289, + "output_tokens": 4457, + "total_tokens": 214746, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 157.80155682563782, + "agent_execution_time": 157.80155682563782, + "token_usage": { + "input_tokens": 596844, + "output_tokens": 3893, + "total_tokens": 600737, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 162.97, + "avg_input_tokens": 429056.5, + "avg_output_tokens": 5060.0, + "avg_total_tokens": 434116.5, + "avg_turn_count": 12.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 162.294846534729, + "agent_execution_time": 162.294846534729, + "token_usage": { + "input_tokens": 554015, + "output_tokens": 3696, + "total_tokens": 557711, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-2", + "success": false, + "execution_time": 360.9465434551239, + "agent_execution_time": 360.9465434551239, + "token_usage": { + "input_tokens": 1365664, + "output_tokens": 6453, + "total_tokens": 1372117, + "reasoning_tokens": 0 + }, + "turn_count": 45 + }, + { + "run": "run-3", + "success": false, + "execution_time": 865.7514960765839, + "agent_execution_time": 865.7514960765839, + "token_usage": { + "input_tokens": 782817, + "output_tokens": 7515, + "total_tokens": 790332, + "reasoning_tokens": 0 + }, + "turn_count": 33 + }, + { + "run": "run-4", + "success": false, + "execution_time": 548.4918584823608, + "agent_execution_time": 548.4918584823608, + "token_usage": { + "input_tokens": 2457548, + "output_tokens": 7775, + "total_tokens": 2465323, + "reasoning_tokens": 0 + }, + "turn_count": 49 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 484.37, + "avg_input_tokens": 1290011.0, + "avg_output_tokens": 6359.8, + "avg_total_tokens": 1296370.8, + "avg_turn_count": 38.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 357.8434314727783, + "agent_execution_time": 357.8434314727783, + "token_usage": { + "input_tokens": 956084, + "output_tokens": 3326, + "total_tokens": 959410, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + { + "run": "run-2", + "success": false, + "execution_time": 207.4619016647339, + "agent_execution_time": 207.4619016647339, + "token_usage": { + "input_tokens": 402296, + "output_tokens": 3159, + "total_tokens": 405455, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": false, + "execution_time": 420.1193721294403, + "agent_execution_time": 420.1193721294403, + "token_usage": { + "input_tokens": 1969206, + "output_tokens": 3855, + "total_tokens": 1973061, + "reasoning_tokens": 0 + }, + "turn_count": 38 + }, + { + "run": "run-4", + "success": false, + "execution_time": 291.2355492115021, + "agent_execution_time": 291.2355492115021, + "token_usage": { + "input_tokens": 675882, + "output_tokens": 2796, + "total_tokens": 678678, + "reasoning_tokens": 0 + }, + "turn_count": 28 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 319.17, + "avg_input_tokens": 1000867.0, + "avg_output_tokens": 3284.0, + "avg_total_tokens": 1004151.0, + "avg_turn_count": 30.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 98.02617001533508, + "agent_execution_time": 98.02617001533508, + "token_usage": { + "input_tokens": 250330, + "output_tokens": 8864, + "total_tokens": 259194, + "reasoning_tokens": 7775 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 143.87897062301636, + "agent_execution_time": 143.87897062301636, + "token_usage": { + "input_tokens": 393034, + "output_tokens": 13363, + "total_tokens": 406397, + "reasoning_tokens": 11872 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 546.5974888801575, + "agent_execution_time": 546.5974888801575, + "token_usage": { + "input_tokens": 247467, + "output_tokens": 101375, + "total_tokens": 348842, + "reasoning_tokens": 5100 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 18.57643699645996, + "agent_execution_time": 18.57643699645996, + "token_usage": { + "input_tokens": 23417, + "output_tokens": 1419, + "total_tokens": 24836, + "reasoning_tokens": 1269 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 201.77, + "avg_input_tokens": 228562.0, + "avg_output_tokens": 31255.2, + "avg_total_tokens": 259817.2, + "avg_turn_count": 11.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/student_database__duplicate_name.json b/task_results/student_database__duplicate_name.json index 2b4af953..76f9fecb 100644 --- a/task_results/student_database__duplicate_name.json +++ b/task_results/student_database__duplicate_name.json @@ -2,6 +2,73 @@ "task": "student_database__duplicate_name", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 282.1767854690552, + "agent_execution_time": 282.1767854690552, + "token_usage": { + "input_tokens": 199314, + "output_tokens": 5275, + "total_tokens": 204589, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 681.841598033905, + "agent_execution_time": 681.841598033905, + "token_usage": { + "input_tokens": 16785, + "output_tokens": 15730, + "total_tokens": 32515, + "reasoning_tokens": 15183 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": true, + "execution_time": 615.7003190517426, + "agent_execution_time": 615.7003190517426, + "token_usage": { + "input_tokens": 16810, + "output_tokens": 14991, + "total_tokens": 31801, + "reasoning_tokens": 14413 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": true, + "execution_time": 636.8990454673767, + "agent_execution_time": 636.8990454673767, + "token_usage": { + "input_tokens": 51838, + "output_tokens": 17767, + "total_tokens": 69605, + "reasoning_tokens": 17108 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 554.15, + "avg_input_tokens": 71186.8, + "avg_output_tokens": 13440.8, + "avg_total_tokens": 84627.5, + "avg_turn_count": 6.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 73.8231930732727, - "agent_execution_time": 73.8231930732727, - "token_usage": { - "input_tokens": 13960, - "output_tokens": 1167, - "total_tokens": 15127, - "reasoning_tokens": null - }, - "turn_count": 4 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 73.82, - "avg_input_tokens": 13960.0, - "avg_output_tokens": 1167.0, - "avg_total_tokens": 15127.0, - "avg_turn_count": 4.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 30.72661805152893, + "agent_execution_time": 30.72661805152893, + "token_usage": { + "input_tokens": 55254, + "output_tokens": 1379, + "total_tokens": 56633, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 20.064249753952026, + "agent_execution_time": 20.064249753952026, + "token_usage": { + "input_tokens": 15288, + "output_tokens": 823, + "total_tokens": 16111, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": true, + "execution_time": 37.03082346916199, + "agent_execution_time": 37.03082346916199, + "token_usage": { + "input_tokens": 79264, + "output_tokens": 1526, + "total_tokens": 80790, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 35.16418647766113, + "agent_execution_time": 35.16418647766113, + "token_usage": { + "input_tokens": 25075, + "output_tokens": 1205, + "total_tokens": 26280, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 30.75, + "avg_input_tokens": 43720.2, + "avg_output_tokens": 1233.2, + "avg_total_tokens": 44953.5, + "avg_turn_count": 5.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 104.50471830368042, + "agent_execution_time": 104.50471830368042, + "token_usage": { + "input_tokens": 347474, + "output_tokens": 8219, + "total_tokens": 355693, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 36.38350868225098, + "agent_execution_time": 36.38350868225098, + "token_usage": { + "input_tokens": 14319, + "output_tokens": 2589, + "total_tokens": 16908, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 104.50471830368042, + "agent_execution_time": 104.50471830368042, + "token_usage": { + "input_tokens": 347474, + "output_tokens": 8219, + "total_tokens": 355693, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 36.38350868225098, + "agent_execution_time": 36.38350868225098, + "token_usage": { + "input_tokens": 14319, + "output_tokens": 2589, + "total_tokens": 16908, + "reasoning_tokens": 0 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 70.44, + "avg_input_tokens": 180896.5, + "avg_output_tokens": 5404.0, + "avg_total_tokens": 186300.5, + "avg_turn_count": 12.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 197.29585361480713, + "agent_execution_time": 197.29585361480713, + "token_usage": { + "input_tokens": 66217, + "output_tokens": 3887, + "total_tokens": 70104, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 53.68654775619507, + "agent_execution_time": 53.68654775619507, + "token_usage": { + "input_tokens": 22475, + "output_tokens": 786, + "total_tokens": 23261, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 173.56470847129822, + "agent_execution_time": 173.56470847129822, + "token_usage": { + "input_tokens": 14501, + "output_tokens": 4301, + "total_tokens": 18802, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 68.06507134437561, + "agent_execution_time": 68.06507134437561, + "token_usage": { + "input_tokens": 51854, + "output_tokens": 1184, + "total_tokens": 53038, + "reasoning_tokens": 0 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 123.15, + "avg_input_tokens": 38761.8, + "avg_output_tokens": 2539.5, + "avg_total_tokens": 41301.2, + "avg_turn_count": 5.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 32.75327706336975, + "agent_execution_time": 32.75327706336975, + "token_usage": { + "input_tokens": 18430, + "output_tokens": 5714, + "total_tokens": 24144, + "reasoning_tokens": 5231 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": true, + "execution_time": 30.554624319076538, + "agent_execution_time": 30.554624319076538, + "token_usage": { + "input_tokens": 59513, + "output_tokens": 4974, + "total_tokens": 64487, + "reasoning_tokens": 4323 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": true, + "execution_time": 51.164172887802124, + "agent_execution_time": 51.164172887802124, + "token_usage": { + "input_tokens": 30699, + "output_tokens": 7250, + "total_tokens": 37949, + "reasoning_tokens": 6379 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": true, + "execution_time": 64.76227450370789, + "agent_execution_time": 64.76227450370789, + "token_usage": { + "input_tokens": 18207, + "output_tokens": 5498, + "total_tokens": 23705, + "reasoning_tokens": 5015 + }, + "turn_count": 4 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 44.81, + "avg_input_tokens": 31712.2, + "avg_output_tokens": 5859.0, + "avg_total_tokens": 37571.2, + "avg_turn_count": 5.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/student_database__english_talent.json b/task_results/student_database__english_talent.json index 443f6de6..61bd0263 100644 --- a/task_results/student_database__english_talent.json +++ b/task_results/student_database__english_talent.json @@ -2,6 +2,73 @@ "task": "student_database__english_talent", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 853.4310433864594, + "agent_execution_time": 853.4310433864594, + "token_usage": { + "input_tokens": 684997, + "output_tokens": 17670, + "total_tokens": 702667, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 745.1283392906189, + "agent_execution_time": 745.1283392906189, + "token_usage": { + "input_tokens": 22620, + "output_tokens": 17823, + "total_tokens": 40443, + "reasoning_tokens": 13421 + }, + "turn_count": 5 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1314.1261701583862, + "agent_execution_time": 1314.1261701583862, + "token_usage": { + "input_tokens": 28325, + "output_tokens": 17500, + "total_tokens": 45825, + "reasoning_tokens": 13053 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 695.2106280326843, + "agent_execution_time": 695.2106280326843, + "token_usage": { + "input_tokens": 82296, + "output_tokens": 19845, + "total_tokens": 102141, + "reasoning_tokens": 15304 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 901.97, + "avg_input_tokens": 204559.5, + "avg_output_tokens": 18209.5, + "avg_total_tokens": 222769.0, + "avg_turn_count": 10.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 1082.4093272686005, - "agent_execution_time": 1082.4093272686005, - "token_usage": { - "input_tokens": 1498886, - "output_tokens": 21607, - "total_tokens": 1520493, - "reasoning_tokens": null - }, - "turn_count": 29 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 1082.41, - "avg_input_tokens": 1498886.0, - "avg_output_tokens": 21607.0, - "avg_total_tokens": 1520493.0, - "avg_turn_count": 29.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 275.5563473701477, + "agent_execution_time": 275.5563473701477, + "token_usage": { + "input_tokens": 1328835, + "output_tokens": 18869, + "total_tokens": 1347704, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": true, + "execution_time": 218.489684343338, + "agent_execution_time": 218.489684343338, + "token_usage": { + "input_tokens": 943517, + "output_tokens": 17222, + "total_tokens": 960739, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": true, + "execution_time": 225.853257894516, + "agent_execution_time": 225.853257894516, + "token_usage": { + "input_tokens": 808857, + "output_tokens": 17850, + "total_tokens": 826707, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 317.272718667984, + "agent_execution_time": 317.272718667984, + "token_usage": { + "input_tokens": 1063621, + "output_tokens": 18079, + "total_tokens": 1081700, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 259.29, + "avg_input_tokens": 1036207.5, + "avg_output_tokens": 18005.0, + "avg_total_tokens": 1054212.5, + "avg_turn_count": 18.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 334.3310852050781, + "agent_execution_time": 334.3310852050781, + "token_usage": { + "input_tokens": 806248, + "output_tokens": 18446, + "total_tokens": 824694, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 268.6572587490082, + "agent_execution_time": 268.6572587490082, + "token_usage": { + "input_tokens": 1569649, + "output_tokens": 14019, + "total_tokens": 1583668, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + { + "run": "run-3", + "success": false, + "execution_time": 334.3310852050781, + "agent_execution_time": 334.3310852050781, + "token_usage": { + "input_tokens": 806248, + "output_tokens": 18446, + "total_tokens": 824694, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 268.6572587490082, + "agent_execution_time": 268.6572587490082, + "token_usage": { + "input_tokens": 1569649, + "output_tokens": 14019, + "total_tokens": 1583668, + "reasoning_tokens": 0 + }, + "turn_count": 39 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 301.49, + "avg_input_tokens": 1187948.5, + "avg_output_tokens": 16232.5, + "avg_total_tokens": 1204181.0, + "avg_turn_count": 30.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 970.915191411972, + "agent_execution_time": 970.915191411972, + "token_usage": { + "input_tokens": 1131984, + "output_tokens": 18708, + "total_tokens": 1150692, + "reasoning_tokens": 0 + }, + "turn_count": 24 + }, + { + "run": "run-2", + "success": false, + "execution_time": 883.6563920974731, + "agent_execution_time": 883.6563920974731, + "token_usage": { + "input_tokens": 3256993, + "output_tokens": 9641, + "total_tokens": 3266634, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-3", + "success": false, + "execution_time": 189.9031000137329, + "agent_execution_time": 189.9031000137329, + "token_usage": { + "input_tokens": 42658, + "output_tokens": 4369, + "total_tokens": 47027, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 186.04001474380493, + "agent_execution_time": 186.04001474380493, + "token_usage": { + "input_tokens": 173063, + "output_tokens": 3111, + "total_tokens": 176174, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 557.63, + "avg_input_tokens": 1151174.5, + "avg_output_tokens": 8957.2, + "avg_total_tokens": 1160131.8, + "avg_turn_count": 35.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 62.378363847732544, + "agent_execution_time": 62.378363847732544, + "token_usage": { + "input_tokens": 718742, + "output_tokens": 5337, + "total_tokens": 724079, + "reasoning_tokens": 3814 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": false, + "execution_time": 86.13072562217712, + "agent_execution_time": 86.13072562217712, + "token_usage": { + "input_tokens": 293028, + "output_tokens": 15962, + "total_tokens": 308990, + "reasoning_tokens": 1711 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": true, + "execution_time": 123.32902336120605, + "agent_execution_time": 123.32902336120605, + "token_usage": { + "input_tokens": 366463, + "output_tokens": 19229, + "total_tokens": 385692, + "reasoning_tokens": 5750 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 105.57179641723633, + "agent_execution_time": 105.57179641723633, + "token_usage": { + "input_tokens": 169865, + "output_tokens": 14808, + "total_tokens": 184673, + "reasoning_tokens": 1966 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 94.35, + "avg_input_tokens": 387024.5, + "avg_output_tokens": 13834.0, + "avg_total_tokens": 400858.5, + "avg_turn_count": 13.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/student_database__gradebased_score.json b/task_results/student_database__gradebased_score.json index 8b9da219..91dfc513 100644 --- a/task_results/student_database__gradebased_score.json +++ b/task_results/student_database__gradebased_score.json @@ -2,6 +2,73 @@ "task": "student_database__gradebased_score", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 614.1367244720459, + "agent_execution_time": 614.1367244720459, + "token_usage": { + "input_tokens": 792495, + "output_tokens": 12021, + "total_tokens": 804516, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1449.01841878891, + "agent_execution_time": 1449.01841878891, + "token_usage": { + "input_tokens": 114831, + "output_tokens": 34643, + "total_tokens": 149474, + "reasoning_tokens": 23623 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 349.3603434562683, + "agent_execution_time": 349.3603434562683, + "token_usage": { + "input_tokens": 23428, + "output_tokens": 8256, + "total_tokens": 31684, + "reasoning_tokens": 3983 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 327.54914593696594, + "agent_execution_time": 327.54914593696594, + "token_usage": { + "input_tokens": 35565, + "output_tokens": 9291, + "total_tokens": 44856, + "reasoning_tokens": 4939 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 685.02, + "avg_input_tokens": 241579.8, + "avg_output_tokens": 16052.8, + "avg_total_tokens": 257632.5, + "avg_turn_count": 10.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 691.0160908699036, - "agent_execution_time": 691.0160908699036, - "token_usage": { - "input_tokens": 248633, - "output_tokens": 14246, - "total_tokens": 262879, - "reasoning_tokens": null - }, - "turn_count": 11 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 691.02, - "avg_input_tokens": 248633.0, - "avg_output_tokens": 14246.0, - "avg_total_tokens": 262879.0, - "avg_turn_count": 11.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 321.98061299324036, + "agent_execution_time": 321.98061299324036, + "token_usage": { + "input_tokens": 745075, + "output_tokens": 17943, + "total_tokens": 763018, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 314.3202350139618, + "agent_execution_time": 314.3202350139618, + "token_usage": { + "input_tokens": 994063, + "output_tokens": 20029, + "total_tokens": 1014092, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 345.93722653388977, + "agent_execution_time": 345.93722653388977, + "token_usage": { + "input_tokens": 890404, + "output_tokens": 26356, + "total_tokens": 916760, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 214.66334557533264, + "agent_execution_time": 214.66334557533264, + "token_usage": { + "input_tokens": 627392, + "output_tokens": 13631, + "total_tokens": 641023, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 299.23, + "avg_input_tokens": 814233.5, + "avg_output_tokens": 19489.8, + "avg_total_tokens": 833723.2, + "avg_turn_count": 17.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 111.69829726219177, + "agent_execution_time": 111.69829726219177, + "token_usage": { + "input_tokens": 213718, + "output_tokens": 10884, + "total_tokens": 224602, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 119.51101922988892, + "agent_execution_time": 119.51101922988892, + "token_usage": { + "input_tokens": 213412, + "output_tokens": 11001, + "total_tokens": 224413, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 111.69829726219177, + "agent_execution_time": 111.69829726219177, + "token_usage": { + "input_tokens": 213718, + "output_tokens": 10884, + "total_tokens": 224602, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 119.51101922988892, + "agent_execution_time": 119.51101922988892, + "token_usage": { + "input_tokens": 213412, + "output_tokens": 11001, + "total_tokens": 224413, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 115.6, + "avg_input_tokens": 213565.0, + "avg_output_tokens": 10942.5, + "avg_total_tokens": 224507.5, + "avg_turn_count": 8.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 562.4351849555969, + "agent_execution_time": 562.4351849555969, + "token_usage": { + "input_tokens": 446093, + "output_tokens": 11828, + "total_tokens": 457921, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 204.41193294525146, + "agent_execution_time": 204.41193294525146, + "token_usage": { + "input_tokens": 81538, + "output_tokens": 4439, + "total_tokens": 85977, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 707.271048784256, + "agent_execution_time": 707.271048784256, + "token_usage": { + "input_tokens": 406390, + "output_tokens": 12729, + "total_tokens": 419119, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 529.8826360702515, + "agent_execution_time": 529.8826360702515, + "token_usage": { + "input_tokens": 283385, + "output_tokens": 12245, + "total_tokens": 295630, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 501.0, + "avg_input_tokens": 304351.5, + "avg_output_tokens": 10310.2, + "avg_total_tokens": 314661.8, + "avg_turn_count": 14.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 45.984002113342285, + "agent_execution_time": 45.984002113342285, + "token_usage": { + "input_tokens": 93941, + "output_tokens": 8397, + "total_tokens": 102338, + "reasoning_tokens": 1876 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 75.5304503440857, + "agent_execution_time": 75.5304503440857, + "token_usage": { + "input_tokens": 97202, + "output_tokens": 11624, + "total_tokens": 108826, + "reasoning_tokens": 1870 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 47.36026859283447, + "agent_execution_time": 47.36026859283447, + "token_usage": { + "input_tokens": 111324, + "output_tokens": 8113, + "total_tokens": 119437, + "reasoning_tokens": 1955 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 61.18229293823242, + "agent_execution_time": 61.18229293823242, + "token_usage": { + "input_tokens": 55746, + "output_tokens": 9800, + "total_tokens": 65546, + "reasoning_tokens": 3706 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 57.51, + "avg_input_tokens": 89553.2, + "avg_output_tokens": 9483.5, + "avg_total_tokens": 99036.8, + "avg_turn_count": 5.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/team_projects__priority_tasks_table.json b/task_results/team_projects__priority_tasks_table.json index b092da94..7378e255 100644 --- a/task_results/team_projects__priority_tasks_table.json +++ b/task_results/team_projects__priority_tasks_table.json @@ -2,6 +2,73 @@ "task": "team_projects__priority_tasks_table", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 1642.3761992454529, + "agent_execution_time": 1642.3761992454529, + "token_usage": { + "input_tokens": 82585, + "output_tokens": 45525, + "total_tokens": 128110, + "reasoning_tokens": 44169 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 2037.2663569450378, + "agent_execution_time": 2037.2663569450378, + "token_usage": { + "input_tokens": 168450, + "output_tokens": 46370, + "total_tokens": 214820, + "reasoning_tokens": 42965 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 116.3914647102356, + "agent_execution_time": 116.3914647102356, + "token_usage": { + "input_tokens": 118896, + "output_tokens": 1678, + "total_tokens": 120574, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1338.0457293987274, + "agent_execution_time": 1338.0457293987274, + "token_usage": { + "input_tokens": 72536, + "output_tokens": 29352, + "total_tokens": 101888, + "reasoning_tokens": 28150 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 1283.52, + "avg_input_tokens": 110616.8, + "avg_output_tokens": 30731.2, + "avg_total_tokens": 141348.0, + "avg_turn_count": 7.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 200.5851604938507, - "agent_execution_time": 200.5851604938507, - "token_usage": { - "input_tokens": 181232, - "output_tokens": 2920, - "total_tokens": 184152, - "reasoning_tokens": null - }, - "turn_count": 11 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 200.59, - "avg_input_tokens": 181232.0, - "avg_output_tokens": 2920.0, - "avg_total_tokens": 184152.0, - "avg_turn_count": 11.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 47.102660179138184, + "agent_execution_time": 47.102660179138184, + "token_usage": { + "input_tokens": 99242, + "output_tokens": 2090, + "total_tokens": 101332, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": false, + "execution_time": 43.743205070495605, + "agent_execution_time": 43.743205070495605, + "token_usage": { + "input_tokens": 78028, + "output_tokens": 1922, + "total_tokens": 79950, + "reasoning_tokens": 0 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 132.97260212898254, + "agent_execution_time": 132.97260212898254, + "token_usage": { + "input_tokens": 499871, + "output_tokens": 5340, + "total_tokens": 505211, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 138.0257544517517, + "agent_execution_time": 138.0257544517517, + "token_usage": { + "input_tokens": 515224, + "output_tokens": 3768, + "total_tokens": 518992, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 90.46, + "avg_input_tokens": 298091.2, + "avg_output_tokens": 3280.0, + "avg_total_tokens": 301371.2, + "avg_turn_count": 8.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 55.677526235580444, + "agent_execution_time": 55.677526235580444, + "token_usage": { + "input_tokens": 90815, + "output_tokens": 1224, + "total_tokens": 92039, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 86.4115207195282, + "agent_execution_time": 86.4115207195282, + "token_usage": { + "input_tokens": 123527, + "output_tokens": 2071, + "total_tokens": 125598, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 188.60025334358215, + "agent_execution_time": 188.60025334358215, + "token_usage": { + "input_tokens": 151045, + "output_tokens": 437, + "total_tokens": 151482, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 195.64994049072266, + "agent_execution_time": 195.64994049072266, + "token_usage": { + "input_tokens": 296398, + "output_tokens": 4884, + "total_tokens": 301282, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 131.58, + "avg_input_tokens": 165446.2, + "avg_output_tokens": 2154.0, + "avg_total_tokens": 167600.2, + "avg_turn_count": 10.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 130.1109175682068, + "agent_execution_time": 130.1109175682068, + "token_usage": { + "input_tokens": 132169, + "output_tokens": 1666, + "total_tokens": 133835, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 72.6975200176239, + "agent_execution_time": 72.6975200176239, + "token_usage": { + "input_tokens": 111307, + "output_tokens": 1081, + "total_tokens": 112388, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 123.36963844299316, + "agent_execution_time": 123.36963844299316, + "token_usage": { + "input_tokens": 140230, + "output_tokens": 1539, + "total_tokens": 141769, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 131.92403602600098, + "agent_execution_time": 131.92403602600098, + "token_usage": { + "input_tokens": 130080, + "output_tokens": 1702, + "total_tokens": 131782, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 114.53, + "avg_input_tokens": 128446.5, + "avg_output_tokens": 1497.0, + "avg_total_tokens": 129943.5, + "avg_turn_count": 7.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 82.55078721046448, + "agent_execution_time": 82.55078721046448, + "token_usage": { + "input_tokens": 164390, + "output_tokens": 7504, + "total_tokens": 171894, + "reasoning_tokens": 6439 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 132.8717610836029, + "agent_execution_time": 132.8717610836029, + "token_usage": { + "input_tokens": 253259, + "output_tokens": 11315, + "total_tokens": 264574, + "reasoning_tokens": 10007 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 77.95200490951538, + "agent_execution_time": 77.95200490951538, + "token_usage": { + "input_tokens": 322625, + "output_tokens": 7860, + "total_tokens": 330485, + "reasoning_tokens": 6586 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 77.79194188117981, + "agent_execution_time": 77.79194188117981, + "token_usage": { + "input_tokens": 265780, + "output_tokens": 5894, + "total_tokens": 271674, + "reasoning_tokens": 4943 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 92.79, + "avg_input_tokens": 251513.5, + "avg_output_tokens": 8143.2, + "avg_total_tokens": 259656.8, + "avg_turn_count": 14.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/team_projects__swap_tasks.json b/task_results/team_projects__swap_tasks.json index 43285d01..3fbb3958 100644 --- a/task_results/team_projects__swap_tasks.json +++ b/task_results/team_projects__swap_tasks.json @@ -2,6 +2,73 @@ "task": "team_projects__swap_tasks", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 711.5996866226196, + "agent_execution_time": 711.5996866226196, + "token_usage": { + "input_tokens": 104798, + "output_tokens": 18779, + "total_tokens": 123577, + "reasoning_tokens": 17374 + }, + "turn_count": 5 + }, + { + "run": "run-2", + "success": true, + "execution_time": 1616.9762933254242, + "agent_execution_time": 1616.9762933254242, + "token_usage": { + "input_tokens": 207004, + "output_tokens": 40225, + "total_tokens": 247229, + "reasoning_tokens": 34600 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 359.2617642879486, + "agent_execution_time": 359.2617642879486, + "token_usage": { + "input_tokens": 1532728, + "output_tokens": 3552, + "total_tokens": 1536280, + "reasoning_tokens": 0 + }, + "turn_count": 29 + }, + { + "run": "run-4", + "success": true, + "execution_time": 1091.5245559215546, + "agent_execution_time": 1091.5245559215546, + "token_usage": { + "input_tokens": 140480, + "output_tokens": 25509, + "total_tokens": 165989, + "reasoning_tokens": 23757 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 944.84, + "avg_input_tokens": 496252.5, + "avg_output_tokens": 22016.2, + "avg_total_tokens": 518268.8, + "avg_turn_count": 11.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 313.7170798778534, - "agent_execution_time": 313.7170798778534, - "token_usage": { - "input_tokens": 524794, - "output_tokens": 3329, - "total_tokens": 528123, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 313.72, - "avg_input_tokens": 524794.0, - "avg_output_tokens": 3329.0, - "avg_total_tokens": 528123.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 106.19912338256836, + "agent_execution_time": 106.19912338256836, + "token_usage": { + "input_tokens": 219582, + "output_tokens": 4346, + "total_tokens": 223928, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": true, + "execution_time": 88.05694580078125, + "agent_execution_time": 88.05694580078125, + "token_usage": { + "input_tokens": 205850, + "output_tokens": 3919, + "total_tokens": 209769, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": true, + "execution_time": 699.3016955852509, + "agent_execution_time": 699.3016955852509, + "token_usage": { + "input_tokens": 287309, + "output_tokens": 3144, + "total_tokens": 290453, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": true, + "execution_time": 109.31528568267822, + "agent_execution_time": 109.31528568267822, + "token_usage": { + "input_tokens": 180459, + "output_tokens": 3606, + "total_tokens": 184065, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 250.72, + "avg_input_tokens": 223300.0, + "avg_output_tokens": 3753.8, + "avg_total_tokens": 227053.8, + "avg_turn_count": 7.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 91.3856053352356, + "agent_execution_time": 91.3856053352356, + "token_usage": { + "input_tokens": 492084, + "output_tokens": 1762, + "total_tokens": 493846, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 254.1742558479309, + "agent_execution_time": 254.1742558479309, + "token_usage": { + "input_tokens": 1145388, + "output_tokens": 3448, + "total_tokens": 1148836, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + { + "run": "run-3", + "success": true, + "execution_time": 360.072603225708, + "agent_execution_time": 360.072603225708, + "token_usage": { + "input_tokens": 474957, + "output_tokens": 4516, + "total_tokens": 479473, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": true, + "execution_time": 257.3946008682251, + "agent_execution_time": 257.3946008682251, + "token_usage": { + "input_tokens": 1001536, + "output_tokens": 4424, + "total_tokens": 1005960, + "reasoning_tokens": 0 + }, + "turn_count": 25 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 240.76, + "avg_input_tokens": 778491.2, + "avg_output_tokens": 3537.5, + "avg_total_tokens": 782028.8, + "avg_turn_count": 25.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 221.01931285858154, + "agent_execution_time": 221.01931285858154, + "token_usage": { + "input_tokens": 608149, + "output_tokens": 2415, + "total_tokens": 610564, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 214.38651371002197, + "agent_execution_time": 214.38651371002197, + "token_usage": { + "input_tokens": 310104, + "output_tokens": 3986, + "total_tokens": 314090, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 290.3823037147522, + "agent_execution_time": 290.3823037147522, + "token_usage": { + "input_tokens": 707521, + "output_tokens": 3228, + "total_tokens": 710749, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": true, + "execution_time": 256.54919052124023, + "agent_execution_time": 256.54919052124023, + "token_usage": { + "input_tokens": 598818, + "output_tokens": 2966, + "total_tokens": 601784, + "reasoning_tokens": 0 + }, + "turn_count": 20 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 245.58, + "avg_input_tokens": 556148.0, + "avg_output_tokens": 3148.8, + "avg_total_tokens": 559296.8, + "avg_turn_count": 18.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 97.05389833450317, + "agent_execution_time": 97.05389833450317, + "token_usage": { + "input_tokens": 430767, + "output_tokens": 7660, + "total_tokens": 438427, + "reasoning_tokens": 5036 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 92.11198449134827, + "agent_execution_time": 92.11198449134827, + "token_usage": { + "input_tokens": 741180, + "output_tokens": 6184, + "total_tokens": 747364, + "reasoning_tokens": 4620 + }, + "turn_count": 15 + }, + { + "run": "run-3", + "success": false, + "execution_time": 233.5587673187256, + "agent_execution_time": 233.5587673187256, + "token_usage": { + "input_tokens": 2995570, + "output_tokens": 13676, + "total_tokens": 3009246, + "reasoning_tokens": 7964 + }, + "turn_count": 23 + }, + { + "run": "run-4", + "success": false, + "execution_time": 95.1660213470459, + "agent_execution_time": 95.1660213470459, + "token_usage": { + "input_tokens": 869336, + "output_tokens": 6109, + "total_tokens": 875445, + "reasoning_tokens": 4064 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 129.47, + "avg_input_tokens": 1259213.2, + "avg_output_tokens": 8407.2, + "avg_total_tokens": 1267620.5, + "avg_turn_count": 16.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/threestudio__code_locating.json b/task_results/threestudio__code_locating.json index 6f7ba06f..e83450c3 100644 --- a/task_results/threestudio__code_locating.json +++ b/task_results/threestudio__code_locating.json @@ -2,6 +2,73 @@ "task": "threestudio__code_locating", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 89.58234930038452, + "agent_execution_time": 89.58234930038452, + "token_usage": { + "input_tokens": 57810, + "output_tokens": 1025, + "total_tokens": 58835, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": true, + "execution_time": 399.82364892959595, + "agent_execution_time": 399.82364892959595, + "token_usage": { + "input_tokens": 57073, + "output_tokens": 8754, + "total_tokens": 65827, + "reasoning_tokens": 7933 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 39.92405438423157, + "agent_execution_time": 39.92405438423157, + "token_usage": { + "input_tokens": 3075, + "output_tokens": 970, + "total_tokens": 4045, + "reasoning_tokens": 936 + }, + "turn_count": 1 + }, + { + "run": "run-4", + "success": true, + "execution_time": 197.1846022605896, + "agent_execution_time": 197.1846022605896, + "token_usage": { + "input_tokens": 38632, + "output_tokens": 4866, + "total_tokens": 43498, + "reasoning_tokens": 4450 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 181.63, + "avg_input_tokens": 39147.5, + "avg_output_tokens": 3903.8, + "avg_total_tokens": 43051.2, + "avg_turn_count": 8.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 1.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 114.27874207496643, - "agent_execution_time": 114.27874207496643, - "token_usage": { - "input_tokens": 47564, - "output_tokens": 1408, - "total_tokens": 48972, - "reasoning_tokens": null - }, - "turn_count": 10 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 114.28, - "avg_input_tokens": 47564.0, - "avg_output_tokens": 1408.0, - "avg_total_tokens": 48972.0, - "avg_turn_count": 10.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 1.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 46.14034676551819, + "agent_execution_time": 46.14034676551819, + "token_usage": { + "input_tokens": 153442, + "output_tokens": 1575, + "total_tokens": 155017, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": true, + "execution_time": 31.756644248962402, + "agent_execution_time": 31.756644248962402, + "token_usage": { + "input_tokens": 101496, + "output_tokens": 982, + "total_tokens": 102478, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": true, + "execution_time": 24.856565237045288, + "agent_execution_time": 24.856565237045288, + "token_usage": { + "input_tokens": 72656, + "output_tokens": 792, + "total_tokens": 73448, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": true, + "execution_time": 31.87791347503662, + "agent_execution_time": 31.87791347503662, + "token_usage": { + "input_tokens": 89543, + "output_tokens": 1010, + "total_tokens": 90553, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 33.66, + "avg_input_tokens": 104284.2, + "avg_output_tokens": 1089.8, + "avg_total_tokens": 105374.0, + "avg_turn_count": 7.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 1.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 26.136942386627197, + "agent_execution_time": 26.136942386627197, + "token_usage": { + "input_tokens": 51573, + "output_tokens": 1646, + "total_tokens": 53219, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": true, + "execution_time": 17.75987672805786, + "agent_execution_time": 17.75987672805786, + "token_usage": { + "input_tokens": 32672, + "output_tokens": 1174, + "total_tokens": 33846, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 26.136942386627197, + "agent_execution_time": 26.136942386627197, + "token_usage": { + "input_tokens": 51573, + "output_tokens": 1646, + "total_tokens": 53219, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-4", + "success": true, + "execution_time": 17.75987672805786, + "agent_execution_time": 17.75987672805786, + "token_usage": { + "input_tokens": 32672, + "output_tokens": 1174, + "total_tokens": 33846, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 21.95, + "avg_input_tokens": 42122.5, + "avg_output_tokens": 1410.0, + "avg_total_tokens": 43532.5, + "avg_turn_count": 11.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 1.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 96.89892864227295, + "agent_execution_time": 96.89892864227295, + "token_usage": { + "input_tokens": 203505, + "output_tokens": 1032, + "total_tokens": 204537, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": true, + "execution_time": 75.37726497650146, + "agent_execution_time": 75.37726497650146, + "token_usage": { + "input_tokens": 65240, + "output_tokens": 769, + "total_tokens": 66009, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 79.04541826248169, + "agent_execution_time": 79.04541826248169, + "token_usage": { + "input_tokens": 99460, + "output_tokens": 618, + "total_tokens": 100078, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": true, + "execution_time": 84.73768305778503, + "agent_execution_time": 84.73768305778503, + "token_usage": { + "input_tokens": 81940, + "output_tokens": 1045, + "total_tokens": 82985, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 84.01, + "avg_input_tokens": 112536.2, + "avg_output_tokens": 866.0, + "avg_total_tokens": 113402.2, + "avg_turn_count": 11.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 20.748981714248657, + "agent_execution_time": 20.748981714248657, + "token_usage": { + "input_tokens": 76010, + "output_tokens": 1570, + "total_tokens": 77580, + "reasoning_tokens": 988 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": true, + "execution_time": 16.69061779975891, + "agent_execution_time": 16.69061779975891, + "token_usage": { + "input_tokens": 43656, + "output_tokens": 1706, + "total_tokens": 45362, + "reasoning_tokens": 1025 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": true, + "execution_time": 18.263986349105835, + "agent_execution_time": 18.263986349105835, + "token_usage": { + "input_tokens": 31888, + "output_tokens": 939, + "total_tokens": 32827, + "reasoning_tokens": 600 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": true, + "execution_time": 32.649340867996216, + "agent_execution_time": 32.649340867996216, + "token_usage": { + "input_tokens": 155450, + "output_tokens": 1346, + "total_tokens": 156796, + "reasoning_tokens": 866 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 22.09, + "avg_input_tokens": 76751.0, + "avg_output_tokens": 1390.2, + "avg_total_tokens": 78141.2, + "avg_turn_count": 9.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/threestudio__output_analysis.json b/task_results/threestudio__output_analysis.json index 84e6ed5f..5edd0443 100644 --- a/task_results/threestudio__output_analysis.json +++ b/task_results/threestudio__output_analysis.json @@ -2,6 +2,73 @@ "task": "threestudio__output_analysis", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 175.36647152900696, + "agent_execution_time": 175.36647152900696, + "token_usage": { + "input_tokens": 320700, + "output_tokens": 2055, + "total_tokens": 322755, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 466.0471348762512, + "agent_execution_time": 466.0471348762512, + "token_usage": { + "input_tokens": 72578, + "output_tokens": 10920, + "total_tokens": 83498, + "reasoning_tokens": 10055 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 512.9723234176636, + "agent_execution_time": 512.9723234176636, + "token_usage": { + "input_tokens": 41169, + "output_tokens": 11496, + "total_tokens": 52665, + "reasoning_tokens": 11030 + }, + "turn_count": 8 + }, + { + "run": "run-4", + "success": false, + "execution_time": 538.5606427192688, + "agent_execution_time": 538.5606427192688, + "token_usage": { + "input_tokens": 60797, + "output_tokens": 14582, + "total_tokens": 75379, + "reasoning_tokens": 13847 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 423.24, + "avg_input_tokens": 123811.0, + "avg_output_tokens": 9763.2, + "avg_total_tokens": 133574.2, + "avg_turn_count": 11.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 152.56501817703247, - "agent_execution_time": 152.56501817703247, - "token_usage": { - "input_tokens": 130219, - "output_tokens": 1628, - "total_tokens": 131847, - "reasoning_tokens": null - }, - "turn_count": 11 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 152.57, - "avg_input_tokens": 130219.0, - "avg_output_tokens": 1628.0, - "avg_total_tokens": 131847.0, - "avg_turn_count": 11.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 40.25979995727539, + "agent_execution_time": 40.25979995727539, + "token_usage": { + "input_tokens": 94885, + "output_tokens": 1410, + "total_tokens": 96295, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 37.768054723739624, + "agent_execution_time": 37.768054723739624, + "token_usage": { + "input_tokens": 94832, + "output_tokens": 1314, + "total_tokens": 96146, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 34.88453793525696, + "agent_execution_time": 34.88453793525696, + "token_usage": { + "input_tokens": 94442, + "output_tokens": 1011, + "total_tokens": 95453, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-4", + "success": false, + "execution_time": 35.897154569625854, + "agent_execution_time": 35.897154569625854, + "token_usage": { + "input_tokens": 94367, + "output_tokens": 1173, + "total_tokens": 95540, + "reasoning_tokens": 0 + }, + "turn_count": 6 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 37.2, + "avg_input_tokens": 94631.5, + "avg_output_tokens": 1227.0, + "avg_total_tokens": 95858.5, + "avg_turn_count": 6.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 22.722601413726807, + "agent_execution_time": 22.722601413726807, + "token_usage": { + "input_tokens": 77786, + "output_tokens": 1572, + "total_tokens": 79358, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 58.15586066246033, + "agent_execution_time": 58.15586066246033, + "token_usage": { + "input_tokens": 56720, + "output_tokens": 1464, + "total_tokens": 58184, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 22.722601413726807, + "agent_execution_time": 22.722601413726807, + "token_usage": { + "input_tokens": 77786, + "output_tokens": 1572, + "total_tokens": 79358, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 58.15586066246033, + "agent_execution_time": 58.15586066246033, + "token_usage": { + "input_tokens": 56720, + "output_tokens": 1464, + "total_tokens": 58184, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 40.44, + "avg_input_tokens": 67253.0, + "avg_output_tokens": 1518.0, + "avg_total_tokens": 68771.0, + "avg_turn_count": 10.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 178.57989811897278, + "agent_execution_time": 178.57989811897278, + "token_usage": { + "input_tokens": 343452, + "output_tokens": 1928, + "total_tokens": 345380, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": false, + "execution_time": 181.03016757965088, + "agent_execution_time": 181.03016757965088, + "token_usage": { + "input_tokens": 337921, + "output_tokens": 2148, + "total_tokens": 340069, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-3", + "success": false, + "execution_time": 232.0110559463501, + "agent_execution_time": 232.0110559463501, + "token_usage": { + "input_tokens": 259428, + "output_tokens": 1761, + "total_tokens": 261189, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-4", + "success": false, + "execution_time": 104.23010182380676, + "agent_execution_time": 104.23010182380676, + "token_usage": { + "input_tokens": 120890, + "output_tokens": 1255, + "total_tokens": 122145, + "reasoning_tokens": 0 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 173.96, + "avg_input_tokens": 265422.8, + "avg_output_tokens": 1773.0, + "avg_total_tokens": 267195.8, + "avg_turn_count": 18.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 15.648688554763794, + "agent_execution_time": 15.648688554763794, + "token_usage": { + "input_tokens": 55295, + "output_tokens": 1606, + "total_tokens": 56901, + "reasoning_tokens": 961 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 25.291381359100342, + "agent_execution_time": 25.291381359100342, + "token_usage": { + "input_tokens": 60896, + "output_tokens": 2537, + "total_tokens": 63433, + "reasoning_tokens": 1739 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 48.82315731048584, + "agent_execution_time": 48.82315731048584, + "token_usage": { + "input_tokens": 97929, + "output_tokens": 6668, + "total_tokens": 104597, + "reasoning_tokens": 5840 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 26.38133478164673, + "agent_execution_time": 26.38133478164673, + "token_usage": { + "input_tokens": 84549, + "output_tokens": 2716, + "total_tokens": 87265, + "reasoning_tokens": 1882 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 29.04, + "avg_input_tokens": 74667.2, + "avg_output_tokens": 3381.8, + "avg_total_tokens": 78049.0, + "avg_turn_count": 10.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/threestudio__requirements_completion.json b/task_results/threestudio__requirements_completion.json index 53105c11..076b7dd2 100644 --- a/task_results/threestudio__requirements_completion.json +++ b/task_results/threestudio__requirements_completion.json @@ -2,6 +2,73 @@ "task": "threestudio__requirements_completion", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 99.41771006584167, + "agent_execution_time": 99.41771006584167, + "token_usage": { + "input_tokens": 146044, + "output_tokens": 1057, + "total_tokens": 147101, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1047.792384147644, + "agent_execution_time": 1047.792384147644, + "token_usage": { + "input_tokens": 158949, + "output_tokens": 24397, + "total_tokens": 183346, + "reasoning_tokens": 23146 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 763.8339810371399, + "agent_execution_time": 763.8339810371399, + "token_usage": { + "input_tokens": 101466, + "output_tokens": 16985, + "total_tokens": 118451, + "reasoning_tokens": 16034 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 236.0360825061798, + "agent_execution_time": 236.0360825061798, + "token_usage": { + "input_tokens": 32746, + "output_tokens": 5710, + "total_tokens": 38456, + "reasoning_tokens": 5311 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 536.77, + "avg_input_tokens": 109801.2, + "avg_output_tokens": 12037.2, + "avg_total_tokens": 121838.5, + "avg_turn_count": 11.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 254.11752915382385, - "agent_execution_time": 254.11752915382385, - "token_usage": { - "input_tokens": 219559, - "output_tokens": 3564, - "total_tokens": 223123, - "reasoning_tokens": null - }, - "turn_count": 24 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 254.12, - "avg_input_tokens": 219559.0, - "avg_output_tokens": 3564.0, - "avg_total_tokens": 223123.0, - "avg_turn_count": 24.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 96.80678296089172, + "agent_execution_time": 96.80678296089172, + "token_usage": { + "input_tokens": 490331, + "output_tokens": 3051, + "total_tokens": 493382, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": false, + "execution_time": 120.20909237861633, + "agent_execution_time": 120.20909237861633, + "token_usage": { + "input_tokens": 819828, + "output_tokens": 3974, + "total_tokens": 823802, + "reasoning_tokens": 0 + }, + "turn_count": 27 + }, + { + "run": "run-3", + "success": false, + "execution_time": 59.96433162689209, + "agent_execution_time": 59.96433162689209, + "token_usage": { + "input_tokens": 412603, + "output_tokens": 1941, + "total_tokens": 414544, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 138.27081394195557, + "agent_execution_time": 138.27081394195557, + "token_usage": { + "input_tokens": 1028510, + "output_tokens": 4324, + "total_tokens": 1032834, + "reasoning_tokens": 0 + }, + "turn_count": 31 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 103.81, + "avg_input_tokens": 687818.0, + "avg_output_tokens": 3322.5, + "avg_total_tokens": 691140.5, + "avg_turn_count": 22.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 38.238969564437866, + "agent_execution_time": 38.238969564437866, + "token_usage": { + "input_tokens": 246150, + "output_tokens": 1918, + "total_tokens": 248068, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 32.380354166030884, + "agent_execution_time": 32.380354166030884, + "token_usage": { + "input_tokens": 263092, + "output_tokens": 2145, + "total_tokens": 265237, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 38.238969564437866, + "agent_execution_time": 38.238969564437866, + "token_usage": { + "input_tokens": 246150, + "output_tokens": 1918, + "total_tokens": 248068, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 32.380354166030884, + "agent_execution_time": 32.380354166030884, + "token_usage": { + "input_tokens": 263092, + "output_tokens": 2145, + "total_tokens": 265237, + "reasoning_tokens": 0 + }, + "turn_count": 16 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 35.31, + "avg_input_tokens": 254621.0, + "avg_output_tokens": 2031.5, + "avg_total_tokens": 256652.5, + "avg_turn_count": 15.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 117.09667921066284, + "agent_execution_time": 117.09667921066284, + "token_usage": { + "input_tokens": 342964, + "output_tokens": 1278, + "total_tokens": 344242, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 112.46994829177856, + "agent_execution_time": 112.46994829177856, + "token_usage": { + "input_tokens": 121258, + "output_tokens": 1393, + "total_tokens": 122651, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 45.402591705322266, + "agent_execution_time": 45.402591705322266, + "token_usage": { + "input_tokens": 25299, + "output_tokens": 628, + "total_tokens": 25927, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 39.34596085548401, + "agent_execution_time": 39.34596085548401, + "token_usage": { + "input_tokens": 25242, + "output_tokens": 553, + "total_tokens": 25795, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 78.58, + "avg_input_tokens": 128690.8, + "avg_output_tokens": 963.0, + "avg_total_tokens": 129653.8, + "avg_turn_count": 10.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 63.79930901527405, + "agent_execution_time": 63.79930901527405, + "token_usage": { + "input_tokens": 742874, + "output_tokens": 5431, + "total_tokens": 748305, + "reasoning_tokens": 3958 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": false, + "execution_time": 48.032352447509766, + "agent_execution_time": 48.032352447509766, + "token_usage": { + "input_tokens": 352580, + "output_tokens": 5079, + "total_tokens": 357659, + "reasoning_tokens": 4177 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 75.89592266082764, + "agent_execution_time": 75.89592266082764, + "token_usage": { + "input_tokens": 698367, + "output_tokens": 6472, + "total_tokens": 704839, + "reasoning_tokens": 5011 + }, + "turn_count": 25 + }, + { + "run": "run-4", + "success": false, + "execution_time": 69.59379458427429, + "agent_execution_time": 69.59379458427429, + "token_usage": { + "input_tokens": 272796, + "output_tokens": 4494, + "total_tokens": 277290, + "reasoning_tokens": 3472 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 64.33, + "avg_input_tokens": 516654.2, + "avg_output_tokens": 5369.0, + "avg_total_tokens": 522023.2, + "avg_turn_count": 20.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/toronto_guide__change_color.json b/task_results/toronto_guide__change_color.json index 9cbff4d9..bfbdd8fa 100644 --- a/task_results/toronto_guide__change_color.json +++ b/task_results/toronto_guide__change_color.json @@ -2,6 +2,73 @@ "task": "toronto_guide__change_color", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 408.6908829212189, + "agent_execution_time": 408.6908829212189, + "token_usage": { + "input_tokens": 43453, + "output_tokens": 11372, + "total_tokens": 54825, + "reasoning_tokens": 10006 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": false, + "execution_time": 1304.6740472316742, + "agent_execution_time": 1304.6740472316742, + "token_usage": { + "input_tokens": 222048, + "output_tokens": 32273, + "total_tokens": 254321, + "reasoning_tokens": 27184 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 227.67290902137756, + "agent_execution_time": 227.67290902137756, + "token_usage": { + "input_tokens": 318265, + "output_tokens": 3079, + "total_tokens": 321344, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 1064.318662405014, + "agent_execution_time": 1064.318662405014, + "token_usage": { + "input_tokens": 115244, + "output_tokens": 24030, + "total_tokens": 139274, + "reasoning_tokens": 20416 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 751.34, + "avg_input_tokens": 174752.5, + "avg_output_tokens": 17688.5, + "avg_total_tokens": 192441.0, + "avg_turn_count": 9.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 542.5061004161835, - "agent_execution_time": 542.5061004161835, - "token_usage": { - "input_tokens": 703574, - "output_tokens": 6298, - "total_tokens": 709872, - "reasoning_tokens": null - }, - "turn_count": 26 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 542.51, - "avg_input_tokens": 703574.0, - "avg_output_tokens": 6298.0, - "avg_total_tokens": 709872.0, - "avg_turn_count": 26.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 257.97811627388, + "agent_execution_time": 257.97811627388, + "token_usage": { + "input_tokens": 704246, + "output_tokens": 10728, + "total_tokens": 714974, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-2", + "success": true, + "execution_time": 249.97449684143066, + "agent_execution_time": 249.97449684143066, + "token_usage": { + "input_tokens": 596650, + "output_tokens": 9249, + "total_tokens": 605899, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 177.7481129169464, + "agent_execution_time": 177.7481129169464, + "token_usage": { + "input_tokens": 635551, + "output_tokens": 5751, + "total_tokens": 641302, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-4", + "success": false, + "execution_time": 219.6611545085907, + "agent_execution_time": 219.6611545085907, + "token_usage": { + "input_tokens": 442376, + "output_tokens": 7734, + "total_tokens": 450110, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 226.34, + "avg_input_tokens": 594705.8, + "avg_output_tokens": 8365.5, + "avg_total_tokens": 603071.2, + "avg_turn_count": 17.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 316.3442397117615, + "agent_execution_time": 316.3442397117615, + "token_usage": { + "input_tokens": 1248337, + "output_tokens": 7434, + "total_tokens": 1255771, + "reasoning_tokens": 0 + }, + "turn_count": 42 + }, + { + "run": "run-2", + "success": false, + "execution_time": 40.342220306396484, + "agent_execution_time": 40.342220306396484, + "token_usage": { + "input_tokens": 151525, + "output_tokens": 786, + "total_tokens": 152311, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 510.50904059410095, + "agent_execution_time": 510.50904059410095, + "token_usage": { + "input_tokens": 1357240, + "output_tokens": 7605, + "total_tokens": 1364845, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + { + "run": "run-4", + "success": false, + "execution_time": 521.605447769165, + "agent_execution_time": 521.605447769165, + "token_usage": { + "input_tokens": 1591205, + "output_tokens": 10707, + "total_tokens": 1601912, + "reasoning_tokens": 0 + }, + "turn_count": 31 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 347.2, + "avg_input_tokens": 1087076.8, + "avg_output_tokens": 6633.0, + "avg_total_tokens": 1093709.8, + "avg_turn_count": 30.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 317.946799993515, + "agent_execution_time": 317.946799993515, + "token_usage": { + "input_tokens": 593569, + "output_tokens": 4642, + "total_tokens": 598211, + "reasoning_tokens": 0 + }, + "turn_count": 20 + }, + { + "run": "run-2", + "success": false, + "execution_time": 134.48727774620056, + "agent_execution_time": 134.48727774620056, + "token_usage": { + "input_tokens": 372858, + "output_tokens": 1865, + "total_tokens": 374723, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 394.53380060195923, + "agent_execution_time": 394.53380060195923, + "token_usage": { + "input_tokens": 460402, + "output_tokens": 5590, + "total_tokens": 465992, + "reasoning_tokens": 0 + }, + "turn_count": 21 + }, + { + "run": "run-4", + "success": false, + "execution_time": 146.48925971984863, + "agent_execution_time": 146.48925971984863, + "token_usage": { + "input_tokens": 225045, + "output_tokens": 1937, + "total_tokens": 226982, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 248.36, + "avg_input_tokens": 412968.5, + "avg_output_tokens": 3508.5, + "avg_total_tokens": 416477.0, + "avg_turn_count": 17.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 82.14975619316101, + "agent_execution_time": 82.14975619316101, + "token_usage": { + "input_tokens": 229823, + "output_tokens": 7295, + "total_tokens": 237118, + "reasoning_tokens": 5626 + }, + "turn_count": 15 + }, + { + "run": "run-2", + "success": false, + "execution_time": 52.450515270233154, + "agent_execution_time": 52.450515270233154, + "token_usage": { + "input_tokens": 208061, + "output_tokens": 5057, + "total_tokens": 213118, + "reasoning_tokens": 3675 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 97.22592878341675, + "agent_execution_time": 97.22592878341675, + "token_usage": { + "input_tokens": 432263, + "output_tokens": 8678, + "total_tokens": 440941, + "reasoning_tokens": 5761 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 61.066659927368164, + "agent_execution_time": 61.066659927368164, + "token_usage": { + "input_tokens": 222132, + "output_tokens": 5490, + "total_tokens": 227622, + "reasoning_tokens": 3802 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 73.22, + "avg_input_tokens": 273069.8, + "avg_output_tokens": 6630.0, + "avg_total_tokens": 279699.8, + "avg_turn_count": 14.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/toronto_guide__weekend_adventure_planner.json b/task_results/toronto_guide__weekend_adventure_planner.json index 669ed052..912f9f26 100644 --- a/task_results/toronto_guide__weekend_adventure_planner.json +++ b/task_results/toronto_guide__weekend_adventure_planner.json @@ -2,6 +2,73 @@ "task": "toronto_guide__weekend_adventure_planner", "service": "notion", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 589.1192388534546, + "agent_execution_time": 589.1192388534546, + "token_usage": { + "input_tokens": 62560, + "output_tokens": 17328, + "total_tokens": 79888, + "reasoning_tokens": 15194 + }, + "turn_count": 4 + }, + { + "run": "run-2", + "success": true, + "execution_time": 707.2529714107513, + "agent_execution_time": 707.2529714107513, + "token_usage": { + "input_tokens": 62524, + "output_tokens": 18956, + "total_tokens": 81480, + "reasoning_tokens": 16760 + }, + "turn_count": 4 + }, + { + "run": "run-3", + "success": false, + "execution_time": 137.0676281452179, + "agent_execution_time": 137.0676281452179, + "token_usage": { + "input_tokens": 179979, + "output_tokens": 1842, + "total_tokens": 181821, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": true, + "execution_time": 1131.1666383743286, + "agent_execution_time": 1131.1666383743286, + "token_usage": { + "input_tokens": 85989, + "output_tokens": 25167, + "total_tokens": 111156, + "reasoning_tokens": 21616 + }, + "turn_count": 5 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 641.15, + "avg_input_tokens": 97763.0, + "avg_output_tokens": 15823.2, + "avg_total_tokens": 113586.2, + "avg_turn_count": 6.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": true, - "execution_time": 208.82200622558594, - "agent_execution_time": 208.82200622558594, - "token_usage": { - "input_tokens": 151321, - "output_tokens": 3160, - "total_tokens": 154481, - "reasoning_tokens": null - }, - "turn_count": 10 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 1, - "avg_agent_execution_time": 208.82, - "avg_input_tokens": 151321.0, - "avg_output_tokens": 3160.0, - "avg_total_tokens": 154481.0, - "avg_turn_count": 10.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 70.25224924087524, + "agent_execution_time": 70.25224924087524, + "token_usage": { + "input_tokens": 122081, + "output_tokens": 3001, + "total_tokens": 125082, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": true, + "execution_time": 84.13027787208557, + "agent_execution_time": 84.13027787208557, + "token_usage": { + "input_tokens": 280673, + "output_tokens": 3512, + "total_tokens": 284185, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-3", + "success": false, + "execution_time": 138.0076162815094, + "agent_execution_time": 138.0076162815094, + "token_usage": { + "input_tokens": 921051, + "output_tokens": 4131, + "total_tokens": 925182, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 123.71289491653442, + "agent_execution_time": 123.71289491653442, + "token_usage": { + "input_tokens": 556893, + "output_tokens": 4147, + "total_tokens": 561040, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 104.03, + "avg_input_tokens": 470174.5, + "avg_output_tokens": 3697.8, + "avg_total_tokens": 473872.2, + "avg_turn_count": 9.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 89.0496027469635, + "agent_execution_time": 89.0496027469635, + "token_usage": { + "input_tokens": 179357, + "output_tokens": 2434, + "total_tokens": 181791, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 807.9166803359985, + "agent_execution_time": 807.9166803359985, + "token_usage": { + "input_tokens": 5091783, + "output_tokens": 11537, + "total_tokens": 5103320, + "reasoning_tokens": 0 + }, + "turn_count": 100 + }, + { + "run": "run-3", + "success": true, + "execution_time": 139.65669798851013, + "agent_execution_time": 139.65669798851013, + "token_usage": { + "input_tokens": 114812, + "output_tokens": 2683, + "total_tokens": 117495, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 206.51325869560242, + "agent_execution_time": 206.51325869560242, + "token_usage": { + "input_tokens": 679223, + "output_tokens": 3495, + "total_tokens": 682718, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 310.78, + "avg_input_tokens": 1516293.8, + "avg_output_tokens": 5037.2, + "avg_total_tokens": 1521331.0, + "avg_turn_count": 36.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 265.73278069496155, + "agent_execution_time": 265.73278069496155, + "token_usage": { + "input_tokens": 547930, + "output_tokens": 2774, + "total_tokens": 550704, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": false, + "execution_time": 120.23621654510498, + "agent_execution_time": 120.23621654510498, + "token_usage": { + "input_tokens": 162599, + "output_tokens": 2542, + "total_tokens": 165141, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": true, + "execution_time": 163.25672841072083, + "agent_execution_time": 163.25672841072083, + "token_usage": { + "input_tokens": 410864, + "output_tokens": 1940, + "total_tokens": 412804, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": true, + "execution_time": 171.87180161476135, + "agent_execution_time": 171.87180161476135, + "token_usage": { + "input_tokens": 216681, + "output_tokens": 2118, + "total_tokens": 218799, + "reasoning_tokens": 0 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 180.27, + "avg_input_tokens": 334518.5, + "avg_output_tokens": 2343.5, + "avg_total_tokens": 336862.0, + "avg_turn_count": 13.5, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 140.93655252456665, + "agent_execution_time": 140.93655252456665, + "token_usage": { + "input_tokens": 456862, + "output_tokens": 13596, + "total_tokens": 470458, + "reasoning_tokens": 7983 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 74.48811268806458, + "agent_execution_time": 74.48811268806458, + "token_usage": { + "input_tokens": 270007, + "output_tokens": 7599, + "total_tokens": 277606, + "reasoning_tokens": 4867 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 126.54843378067017, + "agent_execution_time": 126.54843378067017, + "token_usage": { + "input_tokens": 392261, + "output_tokens": 13488, + "total_tokens": 405749, + "reasoning_tokens": 8195 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 97.88291597366333, + "agent_execution_time": 97.88291597366333, + "token_usage": { + "input_tokens": 475537, + "output_tokens": 6658, + "total_tokens": 482195, + "reasoning_tokens": 4122 + }, + "turn_count": 26 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 109.96, + "avg_input_tokens": 398666.8, + "avg_output_tokens": 10335.2, + "avg_total_tokens": 409002.0, + "avg_turn_count": 18.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/vectors__dba_vector_analysis.json b/task_results/vectors__dba_vector_analysis.json index b580f3a7..4a743c2f 100644 --- a/task_results/vectors__dba_vector_analysis.json +++ b/task_results/vectors__dba_vector_analysis.json @@ -2,6 +2,73 @@ "task": "vectors__dba_vector_analysis", "service": "postgres", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 474.7970817089081, + "agent_execution_time": 474.7970817089081, + "token_usage": { + "input_tokens": 500603, + "output_tokens": 6968, + "total_tokens": 507571, + "reasoning_tokens": 0 + }, + "turn_count": 37 + }, + { + "run": "run-2", + "success": true, + "execution_time": 607.1480605602264, + "agent_execution_time": 607.1480605602264, + "token_usage": { + "input_tokens": 611079, + "output_tokens": 10136, + "total_tokens": 621215, + "reasoning_tokens": 0 + }, + "turn_count": 39 + }, + { + "run": "run-3", + "success": true, + "execution_time": 381.4060699939728, + "agent_execution_time": 381.4060699939728, + "token_usage": { + "input_tokens": 16683, + "output_tokens": 9772, + "total_tokens": 26455, + "reasoning_tokens": 8464 + }, + "turn_count": 4 + }, + { + "run": "run-4", + "success": false, + "execution_time": 766.287832736969, + "agent_execution_time": 766.287832736969, + "token_usage": { + "input_tokens": 9219, + "output_tokens": 13092, + "total_tokens": 22311, + "reasoning_tokens": 12792 + }, + "turn_count": 3 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 557.41, + "avg_input_tokens": 284396.0, + "avg_output_tokens": 9992.0, + "avg_total_tokens": 294388.0, + "avg_turn_count": 20.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 352.9074103832245, - "agent_execution_time": 352.9074103832245, - "token_usage": { - "input_tokens": 441117, - "output_tokens": 8585, - "total_tokens": 449702, - "reasoning_tokens": null - }, - "turn_count": 27 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 352.91, - "avg_input_tokens": 441117.0, - "avg_output_tokens": 8585.0, - "avg_total_tokens": 449702.0, - "avg_turn_count": 27.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 341.4015967845917, + "agent_execution_time": 341.4015967845917, + "token_usage": { + "input_tokens": 711626, + "output_tokens": 17687, + "total_tokens": 729313, + "reasoning_tokens": 0 + }, + "turn_count": 32 + }, + { + "run": "run-2", + "success": true, + "execution_time": 252.53261613845825, + "agent_execution_time": 252.53261613845825, + "token_usage": { + "input_tokens": 499625, + "output_tokens": 14030, + "total_tokens": 513655, + "reasoning_tokens": 0 + }, + "turn_count": 23 + }, + { + "run": "run-3", + "success": true, + "execution_time": 299.88823342323303, + "agent_execution_time": 299.88823342323303, + "token_usage": { + "input_tokens": 1123777, + "output_tokens": 13947, + "total_tokens": 1137724, + "reasoning_tokens": 0 + }, + "turn_count": 31 + }, + { + "run": "run-4", + "success": true, + "execution_time": 294.0345697402954, + "agent_execution_time": 294.0345697402954, + "token_usage": { + "input_tokens": 929872, + "output_tokens": 14959, + "total_tokens": 944831, + "reasoning_tokens": 0 + }, + "turn_count": 35 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 296.96, + "avg_input_tokens": 816225.0, + "avg_output_tokens": 15155.8, + "avg_total_tokens": 831380.8, + "avg_turn_count": 30.25, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 367.1814503669739, + "agent_execution_time": 367.1814503669739, + "token_usage": { + "input_tokens": 378662, + "output_tokens": 6271, + "total_tokens": 384933, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-2", + "success": true, + "execution_time": 398.6988573074341, + "agent_execution_time": 398.6988573074341, + "token_usage": { + "input_tokens": 370868, + "output_tokens": 5668, + "total_tokens": 376536, + "reasoning_tokens": 0 + }, + "turn_count": 30 + }, + { + "run": "run-3", + "success": true, + "execution_time": 258.43459033966064, + "agent_execution_time": 258.43459033966064, + "token_usage": { + "input_tokens": 445683, + "output_tokens": 7381, + "total_tokens": 453064, + "reasoning_tokens": 0 + }, + "turn_count": 34 + }, + { + "run": "run-4", + "success": true, + "execution_time": 439.8751804828644, + "agent_execution_time": 439.8751804828644, + "token_usage": { + "input_tokens": 519258, + "output_tokens": 8700, + "total_tokens": 527958, + "reasoning_tokens": 0 + }, + "turn_count": 36 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 366.05, + "avg_input_tokens": 428617.8, + "avg_output_tokens": 7005.0, + "avg_total_tokens": 435622.8, + "avg_turn_count": 32.5, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 413.6838562488556, + "agent_execution_time": 413.6838562488556, + "token_usage": { + "input_tokens": 463477, + "output_tokens": 5439, + "total_tokens": 468916, + "reasoning_tokens": 0 + }, + "turn_count": 36 + }, + { + "run": "run-2", + "success": true, + "execution_time": 470.38009667396545, + "agent_execution_time": 470.38009667396545, + "token_usage": { + "input_tokens": 459903, + "output_tokens": 7276, + "total_tokens": 467179, + "reasoning_tokens": 0 + }, + "turn_count": 35 + }, + { + "run": "run-3", + "success": true, + "execution_time": 523.7290132045746, + "agent_execution_time": 523.7290132045746, + "token_usage": { + "input_tokens": 691900, + "output_tokens": 7239, + "total_tokens": 699139, + "reasoning_tokens": 0 + }, + "turn_count": 46 + }, + { + "run": "run-4", + "success": false, + "execution_time": 8.236953020095825, + "agent_execution_time": 8.236953020095825, + "token_usage": { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "reasoning_tokens": 0 + }, + "turn_count": 0 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 3, + "avg_agent_execution_time": 354.01, + "avg_input_tokens": 403820.0, + "avg_output_tokens": 4988.5, + "avg_total_tokens": 408808.5, + "avg_turn_count": 29.25, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 58.26984977722168, + "agent_execution_time": 58.26984977722168, + "token_usage": { + "input_tokens": 232828, + "output_tokens": 5869, + "total_tokens": 238697, + "reasoning_tokens": 3992 + }, + "turn_count": 25 + }, + { + "run": "run-2", + "success": true, + "execution_time": 100.25369143486023, + "agent_execution_time": 100.25369143486023, + "token_usage": { + "input_tokens": 256621, + "output_tokens": 7495, + "total_tokens": 264116, + "reasoning_tokens": 4913 + }, + "turn_count": 29 + }, + { + "run": "run-3", + "success": true, + "execution_time": 68.00967717170715, + "agent_execution_time": 68.00967717170715, + "token_usage": { + "input_tokens": 228155, + "output_tokens": 6086, + "total_tokens": 234241, + "reasoning_tokens": 3317 + }, + "turn_count": 26 + }, + { + "run": "run-4", + "success": true, + "execution_time": 49.86398458480835, + "agent_execution_time": 49.86398458480835, + "token_usage": { + "input_tokens": 132589, + "output_tokens": 5718, + "total_tokens": 138307, + "reasoning_tokens": 2610 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 69.1, + "avg_input_tokens": 212548.2, + "avg_output_tokens": 6292.0, + "avg_total_tokens": 218840.2, + "avg_turn_count": 24.75, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/votenet__dataset_comparison.json b/task_results/votenet__dataset_comparison.json index 14c8c92b..91e04e7f 100644 --- a/task_results/votenet__dataset_comparison.json +++ b/task_results/votenet__dataset_comparison.json @@ -2,6 +2,73 @@ "task": "votenet__dataset_comparison", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 169.3777139186859, + "agent_execution_time": 169.3777139186859, + "token_usage": { + "input_tokens": 332261, + "output_tokens": 1554, + "total_tokens": 333815, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 567.3579223155975, + "agent_execution_time": 567.3579223155975, + "token_usage": { + "input_tokens": 91057, + "output_tokens": 12855, + "total_tokens": 103912, + "reasoning_tokens": 12189 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 1373.9668321609497, + "agent_execution_time": 1373.9668321609497, + "token_usage": { + "input_tokens": 206322, + "output_tokens": 33289, + "total_tokens": 239611, + "reasoning_tokens": 32258 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 149.29404830932617, + "agent_execution_time": 149.29404830932617, + "token_usage": { + "input_tokens": 33057, + "output_tokens": 3498, + "total_tokens": 36555, + "reasoning_tokens": 3105 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 565.0, + "avg_input_tokens": 165674.2, + "avg_output_tokens": 12799.0, + "avg_total_tokens": 178473.2, + "avg_turn_count": 11.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 335.7183780670166, - "agent_execution_time": 335.7183780670166, - "token_usage": { - "input_tokens": 567839, - "output_tokens": 4403, - "total_tokens": 572242, - "reasoning_tokens": null - }, - "turn_count": 16 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 335.72, - "avg_input_tokens": 567839.0, - "avg_output_tokens": 4403.0, - "avg_total_tokens": 572242.0, - "avg_turn_count": 16.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 47.45456051826477, + "agent_execution_time": 47.45456051826477, + "token_usage": { + "input_tokens": 181071, + "output_tokens": 1707, + "total_tokens": 182778, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": true, + "execution_time": 104.14181900024414, + "agent_execution_time": 104.14181900024414, + "token_usage": { + "input_tokens": 371285, + "output_tokens": 5042, + "total_tokens": 376327, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": true, + "execution_time": 167.68658542633057, + "agent_execution_time": 167.68658542633057, + "token_usage": { + "input_tokens": 659712, + "output_tokens": 8726, + "total_tokens": 668438, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-4", + "success": false, + "execution_time": 47.27187442779541, + "agent_execution_time": 47.27187442779541, + "token_usage": { + "input_tokens": 220280, + "output_tokens": 1734, + "total_tokens": 222014, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 2, + "avg_agent_execution_time": 91.64, + "avg_input_tokens": 358087.0, + "avg_output_tokens": 4302.2, + "avg_total_tokens": 362389.2, + "avg_turn_count": 11.75, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 34.750967025756836, + "agent_execution_time": 34.750967025756836, + "token_usage": { + "input_tokens": 143277, + "output_tokens": 1019, + "total_tokens": 144296, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 32.7380051612854, + "agent_execution_time": 32.7380051612854, + "token_usage": { + "input_tokens": 181935, + "output_tokens": 1234, + "total_tokens": 183169, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 34.750967025756836, + "agent_execution_time": 34.750967025756836, + "token_usage": { + "input_tokens": 143277, + "output_tokens": 1019, + "total_tokens": 144296, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 32.7380051612854, + "agent_execution_time": 32.7380051612854, + "token_usage": { + "input_tokens": 181935, + "output_tokens": 1234, + "total_tokens": 183169, + "reasoning_tokens": 0 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 33.74, + "avg_input_tokens": 162606.0, + "avg_output_tokens": 1126.5, + "avg_total_tokens": 163732.5, + "avg_turn_count": 7.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 242.0093071460724, + "agent_execution_time": 242.0093071460724, + "token_usage": { + "input_tokens": 824163, + "output_tokens": 2614, + "total_tokens": 826777, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 120.34643077850342, + "agent_execution_time": 120.34643077850342, + "token_usage": { + "input_tokens": 269636, + "output_tokens": 1233, + "total_tokens": 270869, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 148.0965621471405, + "agent_execution_time": 148.0965621471405, + "token_usage": { + "input_tokens": 222337, + "output_tokens": 1135, + "total_tokens": 223472, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 130.5047891139984, + "agent_execution_time": 130.5047891139984, + "token_usage": { + "input_tokens": 224380, + "output_tokens": 1065, + "total_tokens": 225445, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 160.24, + "avg_input_tokens": 385129.0, + "avg_output_tokens": 1511.8, + "avg_total_tokens": 386640.8, + "avg_turn_count": 13.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 117.02768898010254, + "agent_execution_time": 117.02768898010254, + "token_usage": { + "input_tokens": 331066, + "output_tokens": 12728, + "total_tokens": 343794, + "reasoning_tokens": 11964 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 85.15126180648804, + "agent_execution_time": 85.15126180648804, + "token_usage": { + "input_tokens": 400271, + "output_tokens": 8059, + "total_tokens": 408330, + "reasoning_tokens": 7180 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 76.53135967254639, + "agent_execution_time": 76.53135967254639, + "token_usage": { + "input_tokens": 343517, + "output_tokens": 7495, + "total_tokens": 351012, + "reasoning_tokens": 6883 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 102.4737012386322, + "agent_execution_time": 102.4737012386322, + "token_usage": { + "input_tokens": 406958, + "output_tokens": 10420, + "total_tokens": 417378, + "reasoning_tokens": 9592 + }, + "turn_count": 15 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 95.3, + "avg_input_tokens": 370453.0, + "avg_output_tokens": 9675.5, + "avg_total_tokens": 380128.5, + "avg_turn_count": 14.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/votenet__debugging.json b/task_results/votenet__debugging.json index f4521645..7684735b 100644 --- a/task_results/votenet__debugging.json +++ b/task_results/votenet__debugging.json @@ -2,6 +2,73 @@ "task": "votenet__debugging", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 484.8111913204193, + "agent_execution_time": 484.8111913204193, + "token_usage": { + "input_tokens": 640345, + "output_tokens": 8118, + "total_tokens": 648463, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 400.0892035961151, + "agent_execution_time": 400.0892035961151, + "token_usage": { + "input_tokens": 56832, + "output_tokens": 8760, + "total_tokens": 65592, + "reasoning_tokens": 8004 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": false, + "execution_time": 213.07048678398132, + "agent_execution_time": 213.07048678398132, + "token_usage": { + "input_tokens": 34136, + "output_tokens": 5256, + "total_tokens": 39392, + "reasoning_tokens": 4879 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 562.9290969371796, + "agent_execution_time": 562.9290969371796, + "token_usage": { + "input_tokens": 56680, + "output_tokens": 14530, + "total_tokens": 71210, + "reasoning_tokens": 13771 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 415.22, + "avg_input_tokens": 196998.2, + "avg_output_tokens": 9166.0, + "avg_total_tokens": 206164.2, + "avg_turn_count": 12.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 194.19709610939026, - "agent_execution_time": 194.19709610939026, - "token_usage": { - "input_tokens": 90470, - "output_tokens": 2199, - "total_tokens": 92669, - "reasoning_tokens": null - }, - "turn_count": 13 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 194.2, - "avg_input_tokens": 90470.0, - "avg_output_tokens": 2199.0, - "avg_total_tokens": 92669.0, - "avg_turn_count": 13.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 54.12547707557678, + "agent_execution_time": 54.12547707557678, + "token_usage": { + "input_tokens": 143767, + "output_tokens": 2137, + "total_tokens": 145904, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": true, + "execution_time": 55.069950342178345, + "agent_execution_time": 55.069950342178345, + "token_usage": { + "input_tokens": 98578, + "output_tokens": 2417, + "total_tokens": 100995, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-3", + "success": true, + "execution_time": 65.36676406860352, + "agent_execution_time": 65.36676406860352, + "token_usage": { + "input_tokens": 99070, + "output_tokens": 2559, + "total_tokens": 101629, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": true, + "execution_time": 44.07355260848999, + "agent_execution_time": 44.07355260848999, + "token_usage": { + "input_tokens": 63931, + "output_tokens": 1795, + "total_tokens": 65726, + "reasoning_tokens": 0 + }, + "turn_count": 7 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 4, + "avg_agent_execution_time": 54.66, + "avg_input_tokens": 101336.5, + "avg_output_tokens": 2227.0, + "avg_total_tokens": 103563.5, + "avg_turn_count": 9.0, + "pass@4": 1.0, + "pass^4": 1.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 41.36724042892456, + "agent_execution_time": 41.36724042892456, + "token_usage": { + "input_tokens": 47588, + "output_tokens": 1760, + "total_tokens": 49348, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 39.127994775772095, + "agent_execution_time": 39.127994775772095, + "token_usage": { + "input_tokens": 217571, + "output_tokens": 2389, + "total_tokens": 219960, + "reasoning_tokens": 0 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 41.36724042892456, + "agent_execution_time": 41.36724042892456, + "token_usage": { + "input_tokens": 47588, + "output_tokens": 1760, + "total_tokens": 49348, + "reasoning_tokens": 0 + }, + "turn_count": 9 + }, + { + "run": "run-4", + "success": false, + "execution_time": 39.127994775772095, + "agent_execution_time": 39.127994775772095, + "token_usage": { + "input_tokens": 217571, + "output_tokens": 2389, + "total_tokens": 219960, + "reasoning_tokens": 0 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 40.25, + "avg_input_tokens": 132579.5, + "avg_output_tokens": 2074.5, + "avg_total_tokens": 134654.0, + "avg_turn_count": 13.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 232.40335512161255, + "agent_execution_time": 232.40335512161255, + "token_usage": { + "input_tokens": 680975, + "output_tokens": 2442, + "total_tokens": 683417, + "reasoning_tokens": 0 + }, + "turn_count": 26 + }, + { + "run": "run-2", + "success": false, + "execution_time": 117.13639378547668, + "agent_execution_time": 117.13639378547668, + "token_usage": { + "input_tokens": 147555, + "output_tokens": 1338, + "total_tokens": 148893, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 223.7998788356781, + "agent_execution_time": 223.7998788356781, + "token_usage": { + "input_tokens": 247688, + "output_tokens": 2332, + "total_tokens": 250020, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 268.47444128990173, + "agent_execution_time": 268.47444128990173, + "token_usage": { + "input_tokens": 459937, + "output_tokens": 1628, + "total_tokens": 461565, + "reasoning_tokens": 0 + }, + "turn_count": 23 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 210.45, + "avg_input_tokens": 384038.8, + "avg_output_tokens": 1935.0, + "avg_total_tokens": 385973.8, + "avg_turn_count": 19.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": true, + "execution_time": 21.337174654006958, + "agent_execution_time": 21.337174654006958, + "token_usage": { + "input_tokens": 50190, + "output_tokens": 2323, + "total_tokens": 52513, + "reasoning_tokens": 1847 + }, + "turn_count": 9 + }, + { + "run": "run-2", + "success": false, + "execution_time": 33.43651866912842, + "agent_execution_time": 33.43651866912842, + "token_usage": { + "input_tokens": 37346, + "output_tokens": 3773, + "total_tokens": 41119, + "reasoning_tokens": 3235 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 34.03437638282776, + "agent_execution_time": 34.03437638282776, + "token_usage": { + "input_tokens": 67759, + "output_tokens": 3750, + "total_tokens": 71509, + "reasoning_tokens": 3162 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 42.30304169654846, + "agent_execution_time": 42.30304169654846, + "token_usage": { + "input_tokens": 96247, + "output_tokens": 4189, + "total_tokens": 100436, + "reasoning_tokens": 3486 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 1, + "avg_agent_execution_time": 32.78, + "avg_input_tokens": 62885.5, + "avg_output_tokens": 3508.8, + "avg_total_tokens": 66394.2, + "avg_turn_count": 10.0, + "pass@4": 1.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/votenet__requirements_writing.json b/task_results/votenet__requirements_writing.json index 064744c4..c01cecf6 100644 --- a/task_results/votenet__requirements_writing.json +++ b/task_results/votenet__requirements_writing.json @@ -2,6 +2,73 @@ "task": "votenet__requirements_writing", "service": "filesystem", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 174.5799949169159, + "agent_execution_time": 174.5799949169159, + "token_usage": { + "input_tokens": 160787, + "output_tokens": 1960, + "total_tokens": 162747, + "reasoning_tokens": 0 + }, + "turn_count": 22 + }, + { + "run": "run-2", + "success": false, + "execution_time": 625.5320510864258, + "agent_execution_time": 625.5320510864258, + "token_usage": { + "input_tokens": 172927, + "output_tokens": 12635, + "total_tokens": 185562, + "reasoning_tokens": 11729 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 138.60280013084412, + "agent_execution_time": 138.60280013084412, + "token_usage": { + "input_tokens": 29916, + "output_tokens": 3131, + "total_tokens": 33047, + "reasoning_tokens": 2739 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 418.7612109184265, + "agent_execution_time": 418.7612109184265, + "token_usage": { + "input_tokens": 88672, + "output_tokens": 9965, + "total_tokens": 98637, + "reasoning_tokens": 9062 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 339.37, + "avg_input_tokens": 113075.5, + "avg_output_tokens": 6922.8, + "avg_total_tokens": 119998.2, + "avg_turn_count": 11.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 244.1449418067932, - "agent_execution_time": 244.1449418067932, - "token_usage": { - "input_tokens": 372125, - "output_tokens": 2990, - "total_tokens": 375115, - "reasoning_tokens": null - }, - "turn_count": 18 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 244.14, - "avg_input_tokens": 372125.0, - "avg_output_tokens": 2990.0, - "avg_total_tokens": 375115.0, - "avg_turn_count": 18.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 59.96407222747803, + "agent_execution_time": 59.96407222747803, + "token_usage": { + "input_tokens": 402715, + "output_tokens": 2229, + "total_tokens": 404944, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 58.19667339324951, + "agent_execution_time": 58.19667339324951, + "token_usage": { + "input_tokens": 356290, + "output_tokens": 2118, + "total_tokens": 358408, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 52.0265839099884, + "agent_execution_time": 52.0265839099884, + "token_usage": { + "input_tokens": 294978, + "output_tokens": 1965, + "total_tokens": 296943, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 52.09568786621094, + "agent_execution_time": 52.09568786621094, + "token_usage": { + "input_tokens": 231175, + "output_tokens": 1812, + "total_tokens": 232987, + "reasoning_tokens": 0 + }, + "turn_count": 9 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 55.57, + "avg_input_tokens": 321289.5, + "avg_output_tokens": 2031.0, + "avg_total_tokens": 323320.5, + "avg_turn_count": 10.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 51.30520844459534, + "agent_execution_time": 51.30520844459534, + "token_usage": { + "input_tokens": 247922, + "output_tokens": 2129, + "total_tokens": 250051, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-2", + "success": false, + "execution_time": 32.39541029930115, + "agent_execution_time": 32.39541029930115, + "token_usage": { + "input_tokens": 244035, + "output_tokens": 2274, + "total_tokens": 246309, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-3", + "success": false, + "execution_time": 51.30520844459534, + "agent_execution_time": 51.30520844459534, + "token_usage": { + "input_tokens": 247922, + "output_tokens": 2129, + "total_tokens": 250051, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 32.39541029930115, + "agent_execution_time": 32.39541029930115, + "token_usage": { + "input_tokens": 244035, + "output_tokens": 2274, + "total_tokens": 246309, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 41.85, + "avg_input_tokens": 245978.5, + "avg_output_tokens": 2201.5, + "avg_total_tokens": 248180.0, + "avg_turn_count": 18.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 158.32075667381287, + "agent_execution_time": 158.32075667381287, + "token_usage": { + "input_tokens": 142084, + "output_tokens": 1816, + "total_tokens": 143900, + "reasoning_tokens": 0 + }, + "turn_count": 19 + }, + { + "run": "run-2", + "success": false, + "execution_time": 131.65834617614746, + "agent_execution_time": 131.65834617614746, + "token_usage": { + "input_tokens": 95521, + "output_tokens": 1617, + "total_tokens": 97138, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 214.7246491909027, + "agent_execution_time": 214.7246491909027, + "token_usage": { + "input_tokens": 151789, + "output_tokens": 3352, + "total_tokens": 155141, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 161.85045337677002, + "agent_execution_time": 161.85045337677002, + "token_usage": { + "input_tokens": 260578, + "output_tokens": 1482, + "total_tokens": 262060, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 166.64, + "avg_input_tokens": 162493.0, + "avg_output_tokens": 2066.8, + "avg_total_tokens": 164559.8, + "avg_turn_count": 15.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 51.89892864227295, + "agent_execution_time": 51.89892864227295, + "token_usage": { + "input_tokens": 680433, + "output_tokens": 3097, + "total_tokens": 683530, + "reasoning_tokens": 1523 + }, + "turn_count": 28 + }, + { + "run": "run-2", + "success": false, + "execution_time": 71.68468761444092, + "agent_execution_time": 71.68468761444092, + "token_usage": { + "input_tokens": 789156, + "output_tokens": 3974, + "total_tokens": 793130, + "reasoning_tokens": 2217 + }, + "turn_count": 32 + }, + { + "run": "run-3", + "success": false, + "execution_time": 63.722352266311646, + "agent_execution_time": 63.722352266311646, + "token_usage": { + "input_tokens": 853698, + "output_tokens": 3486, + "total_tokens": 857184, + "reasoning_tokens": 1624 + }, + "turn_count": 33 + }, + { + "run": "run-4", + "success": false, + "execution_time": 48.874046087265015, + "agent_execution_time": 48.874046087265015, + "token_usage": { + "input_tokens": 208979, + "output_tokens": 4076, + "total_tokens": 213055, + "reasoning_tokens": 2501 + }, + "turn_count": 8 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 59.05, + "avg_input_tokens": 633066.5, + "avg_output_tokens": 3658.2, + "avg_total_tokens": 636724.8, + "avg_turn_count": 25.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/web_search__birth_of_arvinxu.json b/task_results/web_search__birth_of_arvinxu.json index aaa8342c..622c731a 100644 --- a/task_results/web_search__birth_of_arvinxu.json +++ b/task_results/web_search__birth_of_arvinxu.json @@ -2,6 +2,73 @@ "task": "web_search__birth_of_arvinxu", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 71.80601024627686, + "agent_execution_time": 71.80601024627686, + "token_usage": { + "input_tokens": 119963, + "output_tokens": 722, + "total_tokens": 120685, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-2", + "success": false, + "execution_time": 160.10860681533813, + "agent_execution_time": 160.10860681533813, + "token_usage": { + "input_tokens": 544679, + "output_tokens": 1267, + "total_tokens": 545946, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-3", + "success": false, + "execution_time": 140.54830026626587, + "agent_execution_time": 140.54830026626587, + "token_usage": { + "input_tokens": 204727, + "output_tokens": 1315, + "total_tokens": 206042, + "reasoning_tokens": 0 + }, + "turn_count": 14 + }, + { + "run": "run-4", + "success": false, + "execution_time": 142.61875700950623, + "agent_execution_time": 142.61875700950623, + "token_usage": { + "input_tokens": 415635, + "output_tokens": 1226, + "total_tokens": 416861, + "reasoning_tokens": 0 + }, + "turn_count": 11 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 128.77, + "avg_input_tokens": 321251.0, + "avg_output_tokens": 1132.5, + "avg_total_tokens": 322383.5, + "avg_turn_count": 11.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 202.9762270450592, - "agent_execution_time": 202.9762270450592, - "token_usage": { - "input_tokens": 356654, - "output_tokens": 1465, - "total_tokens": 358119, - "reasoning_tokens": null - }, - "turn_count": 8 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 202.98, - "avg_input_tokens": 356654.0, - "avg_output_tokens": 1465.0, - "avg_total_tokens": 358119.0, - "avg_turn_count": 8.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 103.19063949584961, + "agent_execution_time": 103.19063949584961, + "token_usage": { + "input_tokens": 434800, + "output_tokens": 1896, + "total_tokens": 436696, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-2", + "success": false, + "execution_time": 96.57188034057617, + "agent_execution_time": 96.57188034057617, + "token_usage": { + "input_tokens": 393469, + "output_tokens": 1677, + "total_tokens": 395146, + "reasoning_tokens": 0 + }, + "turn_count": 13 + }, + { + "run": "run-3", + "success": false, + "execution_time": 143.63067388534546, + "agent_execution_time": 143.63067388534546, + "token_usage": { + "input_tokens": 780355, + "output_tokens": 2584, + "total_tokens": 782939, + "reasoning_tokens": 0 + }, + "turn_count": 18 + }, + { + "run": "run-4", + "success": false, + "execution_time": 119.75165700912476, + "agent_execution_time": 119.75165700912476, + "token_usage": { + "input_tokens": 669873, + "output_tokens": 1907, + "total_tokens": 671780, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 115.79, + "avg_input_tokens": 569624.2, + "avg_output_tokens": 2016.0, + "avg_total_tokens": 571640.2, + "avg_turn_count": 14.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 146.43275094032288, + "agent_execution_time": 146.43275094032288, + "token_usage": { + "input_tokens": 664824, + "output_tokens": 1972, + "total_tokens": 666796, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-2", + "success": false, + "execution_time": 116.21653485298157, + "agent_execution_time": 116.21653485298157, + "token_usage": { + "input_tokens": 670862, + "output_tokens": 2334, + "total_tokens": 673196, + "reasoning_tokens": 0 + }, + "turn_count": 16 + }, + { + "run": "run-3", + "success": false, + "execution_time": 52.30887579917908, + "agent_execution_time": 52.30887579917908, + "token_usage": { + "input_tokens": 60062, + "output_tokens": 984, + "total_tokens": 61046, + "reasoning_tokens": 0 + }, + "turn_count": 5 + }, + { + "run": "run-4", + "success": false, + "execution_time": 214.3152141571045, + "agent_execution_time": 214.3152141571045, + "token_usage": { + "input_tokens": 1061338, + "output_tokens": 2960, + "total_tokens": 1064298, + "reasoning_tokens": 0 + }, + "turn_count": 19 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 132.32, + "avg_input_tokens": 614271.5, + "avg_output_tokens": 2062.5, + "avg_total_tokens": 616334.0, + "avg_turn_count": 14.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 159.4384844303131, + "agent_execution_time": 159.4384844303131, + "token_usage": { + "input_tokens": 512478, + "output_tokens": 1521, + "total_tokens": 513999, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-2", + "success": false, + "execution_time": 178.98334622383118, + "agent_execution_time": 178.98334622383118, + "token_usage": { + "input_tokens": 394513, + "output_tokens": 1210, + "total_tokens": 395723, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-3", + "success": false, + "execution_time": 81.99392771720886, + "agent_execution_time": 81.99392771720886, + "token_usage": { + "input_tokens": 141518, + "output_tokens": 830, + "total_tokens": 142348, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 139.45248317718506, + "agent_execution_time": 139.45248317718506, + "token_usage": { + "input_tokens": 328730, + "output_tokens": 1136, + "total_tokens": 329866, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 139.97, + "avg_input_tokens": 344309.8, + "avg_output_tokens": 1174.2, + "avg_total_tokens": 345484.0, + "avg_turn_count": 10.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 66.74845910072327, + "agent_execution_time": 66.74845910072327, + "token_usage": { + "input_tokens": 338317, + "output_tokens": 3629, + "total_tokens": 341946, + "reasoning_tokens": 3092 + }, + "turn_count": 14 + }, + { + "run": "run-2", + "success": false, + "execution_time": 112.36713004112244, + "agent_execution_time": 112.36713004112244, + "token_usage": { + "input_tokens": 956341, + "output_tokens": 4189, + "total_tokens": 960530, + "reasoning_tokens": 3461 + }, + "turn_count": 20 + }, + { + "run": "run-3", + "success": false, + "execution_time": 95.51250839233398, + "agent_execution_time": 95.51250839233398, + "token_usage": { + "input_tokens": 432784, + "output_tokens": 4018, + "total_tokens": 436802, + "reasoning_tokens": 3365 + }, + "turn_count": 16 + }, + { + "run": "run-4", + "success": false, + "execution_time": 129.6201150417328, + "agent_execution_time": 129.6201150417328, + "token_usage": { + "input_tokens": 823807, + "output_tokens": 4200, + "total_tokens": 828007, + "reasoning_tokens": 3553 + }, + "turn_count": 17 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 101.06, + "avg_input_tokens": 637812.2, + "avg_output_tokens": 4009.0, + "avg_total_tokens": 641821.2, + "avg_turn_count": 16.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ { diff --git a/task_results/web_search__r1_arxiv.json b/task_results/web_search__r1_arxiv.json index 8a224afa..ce5262bc 100644 --- a/task_results/web_search__r1_arxiv.json +++ b/task_results/web_search__r1_arxiv.json @@ -2,6 +2,73 @@ "task": "web_search__r1_arxiv", "service": "playwright", "models": { + "deepseek-v3-1-terminus-thinking": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 125.5896155834198, + "agent_execution_time": 125.5896155834198, + "token_usage": { + "input_tokens": 335147, + "output_tokens": 557, + "total_tokens": 335704, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-2", + "success": false, + "execution_time": 144.76980304718018, + "agent_execution_time": 144.76980304718018, + "token_usage": { + "input_tokens": 408530, + "output_tokens": 616, + "total_tokens": 409146, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-3", + "success": false, + "execution_time": 133.10206604003906, + "agent_execution_time": 133.10206604003906, + "token_usage": { + "input_tokens": 284352, + "output_tokens": 506, + "total_tokens": 284858, + "reasoning_tokens": 0 + }, + "turn_count": 7 + }, + { + "run": "run-4", + "success": false, + "execution_time": 201.1830825805664, + "agent_execution_time": 201.1830825805664, + "token_usage": { + "input_tokens": 985933, + "output_tokens": 859, + "total_tokens": 986792, + "reasoning_tokens": 0 + }, + "turn_count": 13 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 151.16, + "avg_input_tokens": 503490.5, + "avg_output_tokens": 634.5, + "avg_total_tokens": 504125.0, + "avg_turn_count": 8.75, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "deepseek-chat": { "runs": [ { @@ -136,32 +203,6 @@ "pass^4": 0.0 } }, - "claude-opus-4-1": { - "runs": [ - { - "run": "run-1", - "success": false, - "execution_time": 192.04109406471252, - "agent_execution_time": 192.04109406471252, - "token_usage": { - "input_tokens": 660052, - "output_tokens": 835, - "total_tokens": 660887, - "reasoning_tokens": null - }, - "turn_count": 7 - } - ], - "summary": { - "total_runs": 1, - "successful_runs": 0, - "avg_agent_execution_time": 192.04, - "avg_input_tokens": 660052.0, - "avg_output_tokens": 835.0, - "avg_total_tokens": 660887.0, - "avg_turn_count": 7.0 - } - }, "glm-4-5": { "runs": [ { @@ -296,6 +337,73 @@ "pass^4": 0.0 } }, + "claude-sonnet-4-5": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 76.32272362709045, + "agent_execution_time": 76.32272362709045, + "token_usage": { + "input_tokens": 477361, + "output_tokens": 1529, + "total_tokens": 478890, + "reasoning_tokens": 0 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 99.96746134757996, + "agent_execution_time": 99.96746134757996, + "token_usage": { + "input_tokens": 817476, + "output_tokens": 1428, + "total_tokens": 818904, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-3", + "success": false, + "execution_time": 93.44523286819458, + "agent_execution_time": 93.44523286819458, + "token_usage": { + "input_tokens": 931601, + "output_tokens": 1443, + "total_tokens": 933044, + "reasoning_tokens": 0 + }, + "turn_count": 12 + }, + { + "run": "run-4", + "success": false, + "execution_time": 81.71313190460205, + "agent_execution_time": 81.71313190460205, + "token_usage": { + "input_tokens": 774558, + "output_tokens": 1249, + "total_tokens": 775807, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 87.86, + "avg_input_tokens": 750249.0, + "avg_output_tokens": 1412.2, + "avg_total_tokens": 751661.2, + "avg_turn_count": 10.5, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "claude-sonnet-4-low": { "runs": [ { @@ -631,6 +739,73 @@ "pass^4": 0.0 } }, + "glm-4-6": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 139.40616869926453, + "agent_execution_time": 139.40616869926453, + "token_usage": { + "input_tokens": 707367, + "output_tokens": 1108, + "total_tokens": 708475, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-2", + "success": false, + "execution_time": 220.23436331748962, + "agent_execution_time": 220.23436331748962, + "token_usage": { + "input_tokens": 731476, + "output_tokens": 4505, + "total_tokens": 735981, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-3", + "success": false, + "execution_time": 204.28891038894653, + "agent_execution_time": 204.28891038894653, + "token_usage": { + "input_tokens": 904752, + "output_tokens": 1548, + "total_tokens": 906300, + "reasoning_tokens": 0 + }, + "turn_count": 15 + }, + { + "run": "run-4", + "success": false, + "execution_time": 139.04804301261902, + "agent_execution_time": 139.04804301261902, + "token_usage": { + "input_tokens": 550098, + "output_tokens": 962, + "total_tokens": 551060, + "reasoning_tokens": 0 + }, + "turn_count": 10 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 175.74, + "avg_input_tokens": 723423.2, + "avg_output_tokens": 2030.8, + "avg_total_tokens": 725454.0, + "avg_turn_count": 11.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "gpt-5-medium": { "runs": [ { @@ -1234,6 +1409,73 @@ "pass^4": 0.0 } }, + "deepseek-v3-1-terminus": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 140.3222622871399, + "agent_execution_time": 140.3222622871399, + "token_usage": { + "input_tokens": 632961, + "output_tokens": 832, + "total_tokens": 633793, + "reasoning_tokens": 0 + }, + "turn_count": 11 + }, + { + "run": "run-2", + "success": false, + "execution_time": 107.73799419403076, + "agent_execution_time": 107.73799419403076, + "token_usage": { + "input_tokens": 206993, + "output_tokens": 445, + "total_tokens": 207438, + "reasoning_tokens": 0 + }, + "turn_count": 6 + }, + { + "run": "run-3", + "success": false, + "execution_time": 166.31330966949463, + "agent_execution_time": 166.31330966949463, + "token_usage": { + "input_tokens": 580182, + "output_tokens": 775, + "total_tokens": 580957, + "reasoning_tokens": 0 + }, + "turn_count": 10 + }, + { + "run": "run-4", + "success": false, + "execution_time": 198.00503611564636, + "agent_execution_time": 198.00503611564636, + "token_usage": { + "input_tokens": 962489, + "output_tokens": 1062, + "total_tokens": 963551, + "reasoning_tokens": 0 + }, + "turn_count": 14 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 153.09, + "avg_input_tokens": 595656.2, + "avg_output_tokens": 778.5, + "avg_total_tokens": 596434.8, + "avg_turn_count": 10.25, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "qwen-3-max": { "runs": [ { @@ -1770,6 +2012,73 @@ "pass^4": 0.0 } }, + "grok-4-fast": { + "runs": [ + { + "run": "run-1", + "success": false, + "execution_time": 53.77536916732788, + "agent_execution_time": 53.77536916732788, + "token_usage": { + "input_tokens": 618356, + "output_tokens": 3224, + "total_tokens": 621580, + "reasoning_tokens": 2392 + }, + "turn_count": 8 + }, + { + "run": "run-2", + "success": false, + "execution_time": 92.56545186042786, + "agent_execution_time": 92.56545186042786, + "token_usage": { + "input_tokens": 1497265, + "output_tokens": 4750, + "total_tokens": 1502015, + "reasoning_tokens": 3580 + }, + "turn_count": 17 + }, + { + "run": "run-3", + "success": false, + "execution_time": 69.89897394180298, + "agent_execution_time": 69.89897394180298, + "token_usage": { + "input_tokens": 916326, + "output_tokens": 4025, + "total_tokens": 920351, + "reasoning_tokens": 3114 + }, + "turn_count": 11 + }, + { + "run": "run-4", + "success": false, + "execution_time": 76.14770317077637, + "agent_execution_time": 76.14770317077637, + "token_usage": { + "input_tokens": 695510, + "output_tokens": 3684, + "total_tokens": 699194, + "reasoning_tokens": 2672 + }, + "turn_count": 12 + } + ], + "summary": { + "total_runs": 4, + "successful_runs": 0, + "avg_agent_execution_time": 73.1, + "avg_input_tokens": 931864.2, + "avg_output_tokens": 3920.8, + "avg_total_tokens": 935785.0, + "avg_turn_count": 12.0, + "pass@4": 0.0, + "pass^4": 0.0 + } + }, "o4-mini": { "runs": [ {