{ "status": "complete", "planned_primary_runs": 72, "completed_primary_runs": 72, "cost_definition": "Frozen API-equivalent token estimate, not OAuth charges.", "time_definition": "Pre-submit/pre-dispatch to final saved assistant response; independent grading excluded.", "limitation": "Base was collected later; provider conditions and cache state may differ. These are descriptive comparisons, not a randomized causal estimate.", "rows": [ { "run": "A1-sunnycode", "task": "A", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T05:15:07.635Z", "final_record_at": "2026-10-14T05:19:04.266Z", "submission_to_final_seconds": 237.104, "main_record_seconds": 236.631, "model_responses": 20, "outer_calls": 23, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "9914c374d6d38fc4f9dbed4bce67bf464c8ace86e96cb330600acb3431513da9", "uncached_input": 33331, "cached_input": 203904, "cache_write": 0, "output": 4100, "estimated_usd": 0.742214, "total_tokens": 241335, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/A1-sunnycode" }, { "run": "A1-sunnycode", "task": "A", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-12T17:04:22.471Z", "final_record_at": "2026-10-12T17:07:33.908Z", "submission_to_final_seconds": 191.732, "main_record_seconds": 191.437, "model_responses": 8, "outer_calls": 7, "nested_operations": 28, "stop_reason": "stop", "source_sha256": "4c75257e4b85438406811c00688dace479bc224349a80a390dbbb9bf673bd781", "uncached_input": 18026, "cached_input": 71040, "cache_write": 0, "output": 3884, "estimated_usd": 0.4455, "total_tokens": 92950, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-publication-v1/runs/A1-sunnycode" }, { "run": "A1-codex", "task": "A", "repeat": 1, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-12T16:45:17.901Z", "final_record_at": "2026-10-12T16:49:11.144Z", "submission_to_final_seconds": 236.861, "main_record_seconds": 233.243, "model_responses": 23, "outer_calls": 22, "nested_operations": null, "stop_reason": "stop", "source_sha256": "fb86ac99dea0052ac95b4cbd9daf9c73c0664f96dfcb739afd2cd66308e80f02", "uncached_input": 65008, "cached_input": 1113728, "cache_write": 0, "output": 5196, "estimated_usd": 2.023608, "total_tokens": 1183932, "arm": "codex", "supplemental": false, "passed": false, "evidence": "pqueue-publication-v1/runs/A1-codex" }, { "run": "A2-sunnycode", "task": "A", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T07:59:12.366Z", "final_record_at": "2026-10-14T08:04:22.787Z", "submission_to_final_seconds": 311.06, "main_record_seconds": 310.421, "model_responses": 19, "outer_calls": 21, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "09e5796783a6a552de8171369953e996a796646421a73e612a35f5eaff6f554b", "uncached_input": 33363, "cached_input": 203264, "cache_write": 0, "output": 4459, "estimated_usd": 0.759844, "total_tokens": 241086, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/A2-sunnycode" }, { "run": "A2-sunnycode", "task": "A", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T02:18:30.486Z", "final_record_at": "2026-10-13T02:22:43.421Z", "submission_to_final_seconds": 253.158, "main_record_seconds": 252.935, "model_responses": 10, "outer_calls": 9, "nested_operations": 26, "stop_reason": "stop", "source_sha256": "50d7050c75591e854eb5669fb7597a0cc8c78f29ea8c06ff4eb1b870f47dc70b", "uncached_input": 16626, "cached_input": 92160, "cache_write": 0, "output": 3827, "estimated_usd": 0.44977, "total_tokens": 112613, "arm": "codemode", "supplemental": false, "passed": false, "evidence": "pqueue-publication-v1/runs/A2-sunnycode" }, { "run": "A2-codex", "task": "A", "repeat": 2, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T02:24:44.268Z", "final_record_at": "2026-10-13T02:27:50.341Z", "submission_to_final_seconds": 197.535, "main_record_seconds": 186.073, "model_responses": 16, "outer_calls": 15, "nested_operations": null, "stop_reason": "stop", "source_sha256": "9e31fc0027b76a8ba3176cef07ead4eb633dd7ad9ebfdf7b67922ec6ecac6298", "uncached_input": 47064, "cached_input": 590464, "cache_write": 0, "output": 3962, "estimated_usd": 1.259204, "total_tokens": 641490, "arm": "codex", "supplemental": false, "passed": false, "evidence": "pqueue-publication-v1/runs/A2-codex" }, { "run": "A3-sunnycode", "task": "A", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T08:05:35.753Z", "final_record_at": "2026-10-14T08:11:24.106Z", "submission_to_final_seconds": 348.856, "main_record_seconds": 348.353, "model_responses": 19, "outer_calls": 22, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "8fe613c5fd1634c8a74b531a7adb29416f08364fb84f0360b1712414cf2a9017", "uncached_input": 38555, "cached_input": 230144, "cache_write": 0, "output": 4298, "estimated_usd": 0.830594, "total_tokens": 272997, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/A3-sunnycode" }, { "run": "A3-sunnycode", "task": "A", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T02:34:56.127Z", "final_record_at": "2026-10-13T02:38:25.412Z", "submission_to_final_seconds": 209.516, "main_record_seconds": 209.285, "model_responses": 7, "outer_calls": 6, "nested_operations": 23, "stop_reason": "stop", "source_sha256": "1c14a68f031238f151c6198957cabc2eca560efd08789f2de3e876c2f2aadd42", "uncached_input": 29299, "cached_input": 40320, "cache_write": 0, "output": 3375, "estimated_usd": 0.50206, "total_tokens": 72994, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-publication-v1/runs/A3-sunnycode" }, { "run": "A3-codex", "task": "A", "repeat": 3, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T02:29:58.408Z", "final_record_at": "2026-10-13T02:33:50.137Z", "submission_to_final_seconds": 243.327, "main_record_seconds": 231.729, "model_responses": 23, "outer_calls": 22, "nested_operations": null, "stop_reason": "stop", "source_sha256": "3064cf29a73a02fbfedcec8e1295dafe5ee0b10f62643ac134b536983791d6ad", "uncached_input": 60868, "cached_input": 1013760, "cache_write": 0, "output": 5071, "estimated_usd": 1.87599, "total_tokens": 1079699, "arm": "codex", "supplemental": false, "passed": true, "evidence": "pqueue-publication-v1/runs/A3-codex" }, { "run": "B1-sunnycode", "task": "B", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T05:21:47.253Z", "final_record_at": "2026-10-14T05:26:55.263Z", "submission_to_final_seconds": 308.382, "main_record_seconds": 308.01, "model_responses": 22, "outer_calls": 23, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "94bf978e458d7ea5a814efdb67d542bc0a8f88a45954c82022ddadf9c2036a6f", "uncached_input": 30464, "cached_input": 381952, "cache_write": 0, "output": 7305, "estimated_usd": 1.051842, "total_tokens": 419721, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/B1-sunnycode" }, { "run": "B1-sunnycode", "task": "B", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-12T17:09:59.262Z", "final_record_at": "2026-10-12T17:14:02.642Z", "submission_to_final_seconds": 244.144, "main_record_seconds": 243.38, "model_responses": 11, "outer_calls": 10, "nested_operations": 24, "stop_reason": "stop", "source_sha256": "95dc1ac747e7271566dd40b92808dbf6d764d343c5f9c132f03c4b54f92234af", "uncached_input": 46919, "cached_input": 142720, "cache_write": 0, "output": 5309, "estimated_usd": 0.87736, "total_tokens": 194948, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-publication-v1/runs/B1-sunnycode" }, { "run": "B1-codex", "task": "B", "repeat": 1, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-12T17:15:34.759Z", "final_record_at": "2026-10-12T17:19:12.357Z", "submission_to_final_seconds": 220.993, "main_record_seconds": 217.598, "model_responses": 19, "outer_calls": 18, "nested_operations": null, "stop_reason": "stop", "source_sha256": "c527f87a048081074dfb7d83ed60775388f824f4e76e454c089c05c817f2f1d7", "uncached_input": 39170, "cached_input": 957824, "cache_write": 0, "output": 6716, "estimated_usd": 1.685324, "total_tokens": 1003710, "arm": "codex", "supplemental": false, "passed": true, "evidence": "pqueue-publication-v1/runs/B1-codex" }, { "run": "B2-sunnycode", "task": "B", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T07:53:10.941Z", "final_record_at": "2026-10-14T07:58:08.072Z", "submission_to_final_seconds": 297.484, "main_record_seconds": 297.131, "model_responses": 17, "outer_calls": 16, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "5e34989f9ca15abccfb0557217a5e7b2dd55d0c19cf1b604445527ebec7012be", "uncached_input": 38512, "cached_input": 196352, "cache_write": 0, "output": 4174, "estimated_usd": 0.790172, "total_tokens": 239038, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/B2-sunnycode" }, { "run": "B2-sunnycode", "task": "B", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T02:12:41.081Z", "final_record_at": "2026-10-13T02:15:47.203Z", "submission_to_final_seconds": 187.192, "main_record_seconds": 186.122, "model_responses": 9, "outer_calls": 8, "nested_operations": 17, "stop_reason": "stop", "source_sha256": "f68b422632fbf6b1d5c10c12b5559a69564441b89352a56dc56631eae9f11631", "uncached_input": 27230, "cached_input": 114304, "cache_write": 0, "output": 4218, "estimated_usd": 0.597504, "total_tokens": 145752, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-publication-v1/runs/B2-sunnycode" }, { "run": "B2-codex", "task": "B", "repeat": 2, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-12T17:21:17.887Z", "final_record_at": "2026-10-12T17:24:43.703Z", "submission_to_final_seconds": 217.103, "main_record_seconds": 205.816, "model_responses": 19, "outer_calls": 18, "nested_operations": null, "stop_reason": "stop", "source_sha256": "9178fe885c9e8d1860954e0ce8f3b0faf614480203b27d658f235a7d55f3c15d", "uncached_input": 57715, "cached_input": 851584, "cache_write": 0, "output": 5827, "estimated_usd": 1.720084, "total_tokens": 915126, "arm": "codex", "supplemental": false, "passed": true, "evidence": "pqueue-publication-v1/runs/B2-codex" }, { "run": "B3-sunnycode", "task": "B", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T08:12:17.642Z", "final_record_at": "2026-10-14T08:15:48.336Z", "submission_to_final_seconds": 210.905, "main_record_seconds": 210.694, "model_responses": 15, "outer_calls": 16, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "0eb82f516674bfe0d565b0a8fd8910002ad3401dab7e527f228a57bec767883c", "uncached_input": 24251, "cached_input": 215296, "cache_write": 0, "output": 4382, "estimated_usd": 0.676906, "total_tokens": 243929, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/B3-sunnycode" }, { "run": "B3-sunnycode", "task": "B", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T02:41:57.420Z", "final_record_at": "2026-10-13T02:44:38.881Z", "submission_to_final_seconds": 162.191, "main_record_seconds": 161.461, "model_responses": 9, "outer_calls": 8, "nested_operations": 17, "stop_reason": "stop", "source_sha256": "b8f5734a8568c9433f58f9047a4b7a3fdecc22178c6b803e598dcdfee7d232f6", "uncached_input": 26542, "cached_input": 113792, "cache_write": 0, "output": 3563, "estimated_usd": 0.557362, "total_tokens": 143897, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-publication-v1/runs/B3-sunnycode" }, { "run": "B3-codex", "task": "B", "repeat": 3, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T02:48:35.307Z", "final_record_at": "2026-10-13T02:53:45.445Z", "submission_to_final_seconds": 322.324, "main_record_seconds": 310.138, "model_responses": 15, "outer_calls": 14, "nested_operations": null, "stop_reason": "stop", "source_sha256": "df177a59b11807e4e1b15ebbac7a8024f70f49a16be3dc0f591ab863af4cecaa", "uncached_input": 59326, "cached_input": 688000, "cache_write": 0, "output": 5964, "estimated_usd": 1.57946, "total_tokens": 753290, "arm": "codex", "supplemental": false, "passed": true, "evidence": "pqueue-publication-v1/runs/B3-codex" }, { "run": "C1-sunnycode", "task": "C", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T05:28:27.771Z", "final_record_at": "2026-10-14T05:43:03.364Z", "submission_to_final_seconds": 876.033, "main_record_seconds": 875.593, "model_responses": 51, "outer_calls": 53, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "c5765033de5edbe299aa59bc3808d96c25b264c02b506506aaa38126736b12f9", "uncached_input": 37443, "cached_input": 1033856, "cache_write": 0, "output": 14272, "estimated_usd": 2.121886, "total_tokens": 1085571, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/C1-sunnycode" }, { "run": "C1-sunnycode", "task": "C", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T03:00:40.530Z", "final_record_at": "2026-10-13T03:08:00.139Z", "submission_to_final_seconds": 440.497, "main_record_seconds": 439.609, "model_responses": 14, "outer_calls": 13, "nested_operations": 44, "stop_reason": "stop", "source_sha256": "b19ac3f210070c2741bcb9fd4a0c9414bb269873916b56021918b85ce0312592", "uncached_input": 59046, "cached_input": 229120, "cache_write": 0, "output": 9649, "estimated_usd": 1.30203, "total_tokens": 297815, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/C1-sunnycode" }, { "run": "C1-codex", "task": "C", "repeat": 1, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T03:30:10.512Z", "final_record_at": "2026-10-13T03:39:54.810Z", "submission_to_final_seconds": 588.709, "main_record_seconds": 584.298, "model_responses": 22, "outer_calls": 21, "nested_operations": null, "stop_reason": "stop", "source_sha256": "3813173693577b83988351da21af55ad153467c310e8ded0324ee6a230b18e4f", "uncached_input": 77760, "cached_input": 1205248, "cache_write": 0, "output": 13534, "estimated_usd": 2.659548, "total_tokens": 1296542, "arm": "codex", "supplemental": false, "passed": false, "evidence": "pqueue-large-extension/runs/C1-codex" }, { "run": "C2-sunnycode", "task": "C", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T07:44:36.628Z", "final_record_at": "2026-10-14T07:51:49.660Z", "submission_to_final_seconds": 433.221, "main_record_seconds": 433.032, "model_responses": 29, "outer_calls": 31, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "a63737c5993529f91243107fe5aebbfde8540a58e8fa634db20d1f1084bfe530", "uncached_input": 62547, "cached_input": 509952, "cache_write": 0, "output": 9456, "estimated_usd": 1.608222, "total_tokens": 581955, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/C2-sunnycode" }, { "run": "C2-sunnycode", "task": "C", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T04:41:51.106Z", "final_record_at": "2026-10-13T04:49:33.898Z", "submission_to_final_seconds": 462.954, "main_record_seconds": 462.792, "model_responses": 16, "outer_calls": 15, "nested_operations": 48, "stop_reason": "stop", "source_sha256": "a90765dc28d8c4859e2e5a6a874fc5ab50c08a66498228ee4bfebcddef0e96f5", "uncached_input": 43623, "cached_input": 347904, "cache_write": 0, "output": 10692, "estimated_usd": 1.318734, "total_tokens": 402219, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/C2-sunnycode" }, { "run": "C2-codex", "task": "C", "repeat": 2, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T04:19:56.871Z", "final_record_at": "2026-10-13T04:27:38.968Z", "submission_to_final_seconds": 465.826, "main_record_seconds": 462.097, "model_responses": 24, "outer_calls": 23, "nested_operations": null, "stop_reason": "stop", "source_sha256": "6d6c49f15c3f740591efb56b2a62aadf6fc2c683ef45e8c228bbda521cd2a5e8", "uncached_input": 73010, "cached_input": 1324416, "cache_write": 0, "output": 13748, "estimated_usd": 2.741916, "total_tokens": 1411174, "arm": "codex", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/C2-codex" }, { "run": "C3-sunnycode", "task": "C", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T08:16:57.314Z", "final_record_at": "2026-10-14T08:25:16.520Z", "submission_to_final_seconds": 499.68, "main_record_seconds": 499.206, "model_responses": 33, "outer_calls": 39, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "f02cafb0c49a25738988072c7cc73d6e1bdd9f40ff66371df36b0cc1d83f80da", "uncached_input": 67866, "cached_input": 650368, "cache_write": 0, "output": 12332, "estimated_usd": 1.945628, "total_tokens": 730566, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/C3-sunnycode" }, { "run": "C3-sunnycode", "task": "C", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T04:52:41.347Z", "final_record_at": "2026-10-13T04:59:38.971Z", "submission_to_final_seconds": 418.146, "main_record_seconds": 417.624, "model_responses": 16, "outer_calls": 15, "nested_operations": 42, "stop_reason": "stop", "source_sha256": "e5f03508fd83fd7bf8da00954623235eaa2a07c2aa0d5dddd51aba162bdb99e7", "uncached_input": 33232, "cached_input": 317312, "cache_write": 0, "output": 9405, "estimated_usd": 1.119882, "total_tokens": 359949, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/C3-sunnycode" }, { "run": "C3-codex", "task": "C", "repeat": 3, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T05:01:31.899Z", "final_record_at": "2026-10-13T05:10:23.388Z", "submission_to_final_seconds": 535.922, "main_record_seconds": 531.489, "model_responses": 27, "outer_calls": 26, "nested_operations": null, "stop_reason": "stop", "source_sha256": "3df66c12a6123a59ea9a937fe8815f4c69bc831778cac56636d7fca9007b9e94", "uncached_input": 80403, "cached_input": 1557888, "cache_write": 0, "output": 12442, "estimated_usd": 2.984018, "total_tokens": 1650733, "arm": "codex", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/C3-codex" }, { "run": "D1-sunnycode", "task": "D", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T05:45:07.925Z", "final_record_at": "2026-10-14T05:48:05.984Z", "submission_to_final_seconds": 178.308, "main_record_seconds": 178.059, "model_responses": 10, "outer_calls": 13, "nested_operations": 0, "stop_reason": "error", "source_sha256": "983de6c4d92f366152741d6a2c5f10330de8483614d7628cbfad2795e2bfa70e", "uncached_input": 27733, "cached_input": 30208, "cache_write": 0, "output": 1777, "estimated_usd": 0.396388, "total_tokens": 59718, "arm": "base", "supplemental": false, "passed": false, "evidence": "three-arm-completion/runs/D1-sunnycode" }, { "run": "D1-sunnycode", "task": "D", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T03:49:30.640Z", "final_record_at": "2026-10-13T04:01:53.077Z", "submission_to_final_seconds": 742.715, "main_record_seconds": 742.437, "model_responses": 16, "outer_calls": 13, "nested_operations": 23, "stop_reason": "stop", "source_sha256": "45d24311adeb46ebcf48ee2ea112eb0ef3af0a3f746007af2159e2209dcd0229", "uncached_input": 30443, "cached_input": 225280, "cache_write": 0, "output": 6577, "estimated_usd": 0.85856, "total_tokens": 262300, "retry_submission_to_final_seconds": 597.331, "auth_retry_protocol_deviation": true, "provider_error_records": [ { "at": "2026-10-13T03:49:32.928Z", "error": "Encountered invalidated oauth token for user, failing request" }, { "at": "2026-10-13T03:55:59.794Z", "error": "terminated" }, { "at": "2026-10-13T03:56:01.832Z", "error": "fetch failed" } ], "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/D1-sunnycode" }, { "run": "D1-codex", "task": "D", "repeat": 1, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T03:42:48.089Z", "final_record_at": "2026-10-13T03:48:12.712Z", "submission_to_final_seconds": 328.572, "main_record_seconds": 324.623, "model_responses": 18, "outer_calls": 17, "nested_operations": null, "stop_reason": "stop", "source_sha256": "1d0a229ce800c64c6df045880d551792de5652c2fab7600edd8b66b1a9479533", "uncached_input": 63821, "cached_input": 878848, "cache_write": 0, "output": 9325, "estimated_usd": 1.983308, "total_tokens": 951994, "arm": "codex", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/D1-codex" }, { "run": "D2-sunnycode", "task": "D", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T07:36:22.981Z", "final_record_at": "2026-10-14T07:43:44.093Z", "submission_to_final_seconds": 441.368, "main_record_seconds": 441.112, "model_responses": 21, "outer_calls": 23, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "2529cb97814f302abd59c8286755d7133004311eafd5a55f1c5f190ef55a8c6a", "uncached_input": 38997, "cached_input": 332928, "cache_write": 0, "output": 7545, "estimated_usd": 1.100148, "total_tokens": 379470, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/D2-sunnycode" }, { "run": "D2-sunnycode", "task": "D", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T04:04:59.020Z", "final_record_at": "2026-10-13T04:09:13.243Z", "submission_to_final_seconds": 254.44, "main_record_seconds": 254.223, "model_responses": 12, "outer_calls": 11, "nested_operations": 25, "stop_reason": "stop", "source_sha256": "b3d941c8a0d993197d2ca17c30afd1317981588eccd3b154f2875538d29bbd80", "uncached_input": 39785, "cached_input": 175232, "cache_write": 0, "output": 6173, "estimated_usd": 0.881732, "total_tokens": 221190, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/D2-sunnycode" }, { "run": "D2-codex", "task": "D", "repeat": 2, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T04:13:42.069Z", "final_record_at": "2026-10-13T04:18:32.634Z", "submission_to_final_seconds": 294.742, "main_record_seconds": 290.565, "model_responses": 19, "outer_calls": 18, "nested_operations": null, "stop_reason": "stop", "source_sha256": "061af8274e9aa3b8ec28f13f20eeffd52493178f7c0ff643378e1725de0babfd", "uncached_input": 62112, "cached_input": 909184, "cache_write": 0, "output": 8578, "estimated_usd": 1.959204, "total_tokens": 979874, "arm": "codex", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/D2-codex" }, { "run": "D3-sunnycode", "task": "D", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T08:26:40.991Z", "final_record_at": "2026-10-14T08:36:07.626Z", "submission_to_final_seconds": 567.208, "main_record_seconds": 566.635, "model_responses": 21, "outer_calls": 25, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "0785d0dfd6b1d0837b07181f0ebaa0d5c07bed43e1a6c65ab8e65d73ed0fc5f7", "uncached_input": 27348, "cached_input": 351488, "cache_write": 0, "output": 6400, "estimated_usd": 0.944968, "total_tokens": 385236, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/D3-sunnycode" }, { "run": "D3-sunnycode", "task": "D", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T05:19:14.796Z", "final_record_at": "2026-10-13T05:24:31.585Z", "submission_to_final_seconds": 316.999, "main_record_seconds": 316.789, "model_responses": 11, "outer_calls": 10, "nested_operations": 28, "stop_reason": "stop", "source_sha256": "637570a7b6a005c98988162b23ee5b5ae173335ccbfb6ce0156c1a04e61eb07c", "uncached_input": 43537, "cached_input": 150400, "cache_write": 0, "output": 6243, "estimated_usd": 0.8979199999999999, "total_tokens": 200180, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/D3-sunnycode" }, { "run": "D3-codex", "task": "D", "repeat": 3, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T05:12:04.354Z", "final_record_at": "2026-10-13T05:16:32.223Z", "submission_to_final_seconds": 271.907, "main_record_seconds": 267.869, "model_responses": 19, "outer_calls": 18, "nested_operations": null, "stop_reason": "stop", "source_sha256": "1eb5ae85427d3e39da8418643c73d2c8a36caa16e1ba90da433cc2bbf2db5cf1", "uncached_input": 39383, "cached_input": 937856, "cache_write": 0, "output": 7748, "estimated_usd": 1.7190860000000001, "total_tokens": 984987, "arm": "codex", "supplemental": false, "passed": true, "evidence": "pqueue-large-extension/runs/D3-codex" }, { "run": "E1-sunnycode", "task": "E", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T05:57:42.699Z", "final_record_at": "2026-10-14T05:58:42.098Z", "submission_to_final_seconds": 60.067, "main_record_seconds": 59.399, "model_responses": 8, "outer_calls": 9, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "264805916953bfff5f90e0f3f72b1fa2818f6cacefb01141bb47452faa3c6825", "uncached_input": 19490, "cached_input": 62464, "cache_write": 0, "output": 1469, "estimated_usd": 0.330814, "total_tokens": 83423, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/E1-sunnycode" }, { "run": "E1-sunnycode", "task": "E", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T18:05:56.965Z", "final_record_at": "2026-10-13T18:06:49.826Z", "submission_to_final_seconds": 53.714, "main_record_seconds": 52.861, "model_responses": 7, "outer_calls": 6, "nested_operations": 13, "stop_reason": "stop", "source_sha256": "ad61198b9158b7261e694f37238560bfefa289c933ebaa07821b162bdb0df010", "uncached_input": 15551, "cached_input": 41984, "cache_write": 0, "output": 1414, "estimated_usd": 0.268194, "total_tokens": 58949, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/E1-sunnycode" }, { "run": "E1-codex", "task": "E", "repeat": 1, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T17:41:53.106Z", "final_record_at": "2026-10-13T17:42:59.211Z", "submission_to_final_seconds": 70.068, "main_record_seconds": 66.105, "model_responses": 8, "outer_calls": 7, "nested_operations": null, "stop_reason": "stop", "source_sha256": "4c089dc75e4e87f312d8cc6ec571a7113825e5075f719cd9bf120934ebd0d7f5", "uncached_input": 22362, "cached_input": 284032, "cache_write": 0, "output": 1712, "estimated_usd": 0.593252, "total_tokens": 308106, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/E1-codex" }, { "run": "E2-sunnycode", "task": "E", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T07:32:54.111Z", "final_record_at": "2026-10-14T07:35:21.113Z", "submission_to_final_seconds": 147.321, "main_record_seconds": 147.002, "model_responses": 12, "outer_calls": 11, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "5ece32a850fcdde882278bcf2d644fea25e878094be4ea9ea9c8e821735f6c4b", "uncached_input": 11310, "cached_input": 80640, "cache_write": 0, "output": 1665, "estimated_usd": 0.27699, "total_tokens": 93615, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/E2-sunnycode" }, { "run": "E2-sunnycode", "task": "E", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:27:15.192Z", "final_record_at": "2026-10-14T03:28:21.565Z", "submission_to_final_seconds": 67.431, "main_record_seconds": 66.373, "model_responses": 8, "outer_calls": 7, "nested_operations": 13, "stop_reason": "stop", "source_sha256": "90ff49aee1fa5e8c592a61fe18d7257efe7ae41e176c6309f47f6d013ffc2895", "uncached_input": 11368, "cached_input": 47104, "cache_write": 0, "output": 1819, "estimated_usd": 0.251734, "total_tokens": 60291, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/E2-sunnycode" }, { "run": "E2-codex", "task": "E", "repeat": 2, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:25:38.578Z", "final_record_at": "2026-10-14T03:26:58.037Z", "submission_to_final_seconds": 84.188, "main_record_seconds": 79.459, "model_responses": 8, "outer_calls": 7, "nested_operations": null, "stop_reason": "stop", "source_sha256": "54fba6d143e57d607a4c8e6f7fef51df33c78fdd0c8729b3cd87a283e62d540e", "uncached_input": 43488, "cached_input": 255872, "cache_write": 0, "output": 1914, "estimated_usd": 0.786452, "total_tokens": 301274, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/E2-codex" }, { "run": "E3-sunnycode", "task": "E", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T08:38:09.715Z", "final_record_at": "2026-10-14T08:39:51.535Z", "submission_to_final_seconds": 103.249, "main_record_seconds": 101.82, "model_responses": 9, "outer_calls": 8, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "bfad201e3e876698e6b07acf456c45ee4ea2052b49e3bc1a4bf6b3e614c5ac1c", "uncached_input": 8799, "cached_input": 65280, "cache_write": 0, "output": 1306, "estimated_usd": 0.21857000000000001, "total_tokens": 75385, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/E3-sunnycode" }, { "run": "E3-sunnycode", "task": "E", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:30:20.707Z", "final_record_at": "2026-10-14T03:31:29.021Z", "submission_to_final_seconds": 69.293, "main_record_seconds": 68.314, "model_responses": 6, "outer_calls": 5, "nested_operations": 12, "stop_reason": "stop", "source_sha256": "025b2454a8d60208c56b065ba1bd789d8ec7ecf1befe86aca3a9eb29cf7b78ee", "uncached_input": 20729, "cached_input": 25600, "cache_write": 0, "output": 1662, "estimated_usd": 0.31599, "total_tokens": 47991, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/E3-sunnycode" }, { "run": "E3-codex", "task": "E", "repeat": 3, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:29:04.828Z", "final_record_at": "2026-10-14T03:30:02.877Z", "submission_to_final_seconds": 64.094, "main_record_seconds": 58.049, "model_responses": 7, "outer_calls": 6, "nested_operations": null, "stop_reason": "stop", "source_sha256": "10c9289e91ab59a8efa468fb01f12866ea7b3dfd31af22df5becb79c301e5618", "uncached_input": 25250, "cached_input": 239360, "cache_write": 0, "output": 1505, "estimated_usd": 0.56711, "total_tokens": 266115, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/E3-codex" }, { "run": "F1-sunnycode", "task": "F", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T06:00:22.960Z", "final_record_at": "2026-10-14T06:02:45.954Z", "submission_to_final_seconds": 143.84, "main_record_seconds": 142.994, "model_responses": 13, "outer_calls": 15, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "4e12bb1b7ec72f3a186ac88498ee3a2893e5f52fd1ccb5d27b6b958e2d3f03ce", "uncached_input": 15890, "cached_input": 125568, "cache_write": 0, "output": 3624, "estimated_usd": 0.465668, "total_tokens": 145082, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/F1-sunnycode" }, { "run": "F1-sunnycode", "task": "F", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T18:07:40.160Z", "final_record_at": "2026-10-13T18:09:26.691Z", "submission_to_final_seconds": 107.311, "main_record_seconds": 106.531, "model_responses": 9, "outer_calls": 8, "nested_operations": 17, "stop_reason": "stop", "source_sha256": "93753ce2be44731078511c2e4225856ea5e9e0db75743cdfe58a9f123d4558f6", "uncached_input": 26087, "cached_input": 73088, "cache_write": 0, "output": 3072, "estimated_usd": 0.487558, "total_tokens": 102247, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/F1-sunnycode" }, { "run": "F1-codex", "task": "F", "repeat": 1, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T18:10:04.628Z", "final_record_at": "2026-10-13T18:11:58.979Z", "submission_to_final_seconds": 118.958, "main_record_seconds": 114.351, "model_responses": 8, "outer_calls": 7, "nested_operations": null, "stop_reason": "stop", "source_sha256": "4d9ec610559094ba564a7be1e7fa0df8fdd8f4bae307243a556245f97284b8dd", "uncached_input": 48158, "cached_input": 279040, "cache_write": 0, "output": 3348, "estimated_usd": 0.9280200000000001, "total_tokens": 330546, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/F1-codex" }, { "run": "F2-sunnycode", "task": "F", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T07:01:32.812Z", "final_record_at": "2026-10-14T07:05:46.380Z", "submission_to_final_seconds": 254.452, "main_record_seconds": 253.568, "model_responses": 14, "outer_calls": 13, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "f7778d5d7b0bd916907bfd195d1fd0ad30d09d20f1d0b0bb1ad47c186ffa9237", "uncached_input": 15932, "cached_input": 132864, "cache_write": 0, "output": 3213, "estimated_usd": 0.452834, "total_tokens": 152009, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/F2-sunnycode" }, { "run": "F2-sunnycode", "task": "F", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:20:58.209Z", "final_record_at": "2026-10-14T03:22:42.533Z", "submission_to_final_seconds": 105.283, "main_record_seconds": 104.324, "model_responses": 8, "outer_calls": 7, "nested_operations": 16, "stop_reason": "stop", "source_sha256": "0bec640848f5d292f4ca0592b6881e58f2034fd4f42975c74bf226c7f33ed955", "uncached_input": 25619, "cached_input": 40704, "cache_write": 0, "output": 3099, "estimated_usd": 0.45184399999999997, "total_tokens": 69422, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/F2-sunnycode" }, { "run": "F2-codex", "task": "F", "repeat": 2, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:23:20.240Z", "final_record_at": "2026-10-14T03:25:12.436Z", "submission_to_final_seconds": 116.883, "main_record_seconds": 112.196, "model_responses": 9, "outer_calls": 8, "nested_operations": null, "stop_reason": "stop", "source_sha256": "895726164f4baf9c5175c92f267d63db5b9ac8bf094991b15cba444c66d21753", "uncached_input": 46029, "cached_input": 308352, "cache_write": 0, "output": 3525, "estimated_usd": 0.944892, "total_tokens": 357906, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/F2-codex" }, { "run": "F3-sunnycode", "task": "F", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T08:40:34.847Z", "final_record_at": "2026-10-14T08:43:12.936Z", "submission_to_final_seconds": 159.311, "main_record_seconds": 158.089, "model_responses": 14, "outer_calls": 16, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "1d3b8e66d7ab0a6be98c06553c321fa95d1a4ef858388e2f0a7a429803c86137", "uncached_input": 30240, "cached_input": 121088, "cache_write": 0, "output": 3385, "estimated_usd": 0.592738, "total_tokens": 154713, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/F3-sunnycode" }, { "run": "F3-sunnycode", "task": "F", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:32:17.457Z", "final_record_at": "2026-10-14T03:34:33.533Z", "submission_to_final_seconds": 136.972, "main_record_seconds": 136.076, "model_responses": 9, "outer_calls": 8, "nested_operations": 20, "stop_reason": "stop", "source_sha256": "38c29f4fc4465f4db4c3c8249f7377eed65b93c25bcd7298a133fd3cb1e303d0", "uncached_input": 27353, "cached_input": 88960, "cache_write": 0, "output": 4215, "estimated_usd": 0.57324, "total_tokens": 120528, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/F3-sunnycode" }, { "run": "F3-codex", "task": "F", "repeat": 3, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:35:09.497Z", "final_record_at": "2026-10-14T03:37:11.428Z", "submission_to_final_seconds": 126.636, "main_record_seconds": 121.931, "model_responses": 10, "outer_calls": 9, "nested_operations": null, "stop_reason": "stop", "source_sha256": "5111566d81740cfc02e1af6ac5161447e0410b565b3a4d8a1cf4c929339bb575", "uncached_input": 25518, "cached_input": 373248, "cache_write": 0, "output": 4105, "estimated_usd": 0.833678, "total_tokens": 402871, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/F3-codex" }, { "run": "G1-sunnycode", "task": "G", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T06:11:49.490Z", "final_record_at": "2026-10-14T06:15:36.885Z", "submission_to_final_seconds": 227.895, "main_record_seconds": 227.395, "model_responses": 16, "outer_calls": 16, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "a29534cf33166456c02918b5659e50456b58eeec087bc69e7f67257a9aef921b", "uncached_input": 27787, "cached_input": 117248, "cache_write": 0, "output": 2852, "estimated_usd": 0.537718, "total_tokens": 147887, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/G1-sunnycode" }, { "run": "G1-sunnycode", "task": "G", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T18:14:58.623Z", "final_record_at": "2026-10-13T18:16:29.579Z", "submission_to_final_seconds": 91.469, "main_record_seconds": 90.956, "model_responses": 8, "outer_calls": 7, "nested_operations": 16, "stop_reason": "stop", "source_sha256": "f13e88480ee116e47687594c1079cf254b595bb88dd3f7ef59d3e1149042aa83", "uncached_input": 26094, "cached_input": 75264, "cache_write": 0, "output": 2525, "estimated_usd": 0.462454, "total_tokens": 103883, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/G1-sunnycode" }, { "run": "G1-codex", "task": "G", "repeat": 1, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T18:12:26.654Z", "final_record_at": "2026-10-13T18:14:36.882Z", "submission_to_final_seconds": 134.349, "main_record_seconds": 130.228, "model_responses": 10, "outer_calls": 9, "nested_operations": null, "stop_reason": "stop", "source_sha256": "444f34c93336a0c495abbdfbf8381b820753099d04e7be4e2ff6020e71524677", "uncached_input": 46924, "cached_input": 356352, "cache_write": 0, "output": 3376, "estimated_usd": 0.994392, "total_tokens": 406652, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/G1-codex" }, { "run": "G2-sunnycode", "task": "G", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T06:39:39.384Z", "final_record_at": "2026-10-14T06:49:26.508Z", "submission_to_final_seconds": 587.798, "main_record_seconds": 587.124, "model_responses": 33, "outer_calls": 17, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "b49c7c5d26a1a8e0c0a9c2dbaed6fd25ec4c8460abc13825bb7e07a8953dbf24", "uncached_input": 19931, "cached_input": 163328, "cache_write": 0, "output": 3249, "estimated_usd": 0.525088, "total_tokens": 186508, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/G2-sunnycode" }, { "run": "G2-sunnycode", "task": "G", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:18:03.396Z", "final_record_at": "2026-10-14T03:20:08.645Z", "submission_to_final_seconds": 125.81, "main_record_seconds": 125.249, "model_responses": 9, "outer_calls": 8, "nested_operations": 20, "stop_reason": "stop", "source_sha256": "e660540003246cd24d58ad7a67ac1cfd408774154be1c0d722925feff1c57130", "uncached_input": 29427, "cached_input": 76800, "cache_write": 0, "output": 3313, "estimated_usd": 0.53672, "total_tokens": 109540, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/G2-sunnycode" }, { "run": "G2-codex", "task": "G", "repeat": 2, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:15:33.195Z", "final_record_at": "2026-10-14T03:17:44.625Z", "submission_to_final_seconds": 136.857, "main_record_seconds": 131.43, "model_responses": 10, "outer_calls": 9, "nested_operations": null, "stop_reason": "stop", "source_sha256": "2995db922daa499de90240d0cdbaa8d4887a40ce9b3ab373efaa81f988f80d77", "uncached_input": 24754, "cached_input": 385408, "cache_write": 0, "output": 3180, "estimated_usd": 0.791948, "total_tokens": 413342, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/G2-codex" }, { "run": "G3-sunnycode", "task": "G", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T08:44:19.277Z", "final_record_at": "2026-10-14T08:46:25.005Z", "submission_to_final_seconds": 126.909, "main_record_seconds": 125.728, "model_responses": 11, "outer_calls": 12, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "73e786371a4518de1fd609e5959e78aaf3e23ded0a1d3aaa936ede18ec464b47", "uncached_input": 15936, "cached_input": 99968, "cache_write": 0, "output": 2422, "estimated_usd": 0.380428, "total_tokens": 118326, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/G3-sunnycode" }, { "run": "G3-sunnycode", "task": "G", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:43:15.703Z", "final_record_at": "2026-10-14T03:44:52.263Z", "submission_to_final_seconds": 97.471, "main_record_seconds": 96.56, "model_responses": 8, "outer_calls": 7, "nested_operations": 16, "stop_reason": "stop", "source_sha256": "447ac67bd4b6b6c75882644c6c3262bb7c809cbdad0e290e5b1d379fd142624d", "uncached_input": 20465, "cached_input": 61952, "cache_write": 0, "output": 2651, "estimated_usd": 0.399152, "total_tokens": 85068, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/G3-sunnycode" }, { "run": "G3-codex", "task": "G", "repeat": 3, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:37:36.633Z", "final_record_at": "2026-10-14T03:39:43.888Z", "submission_to_final_seconds": 134.384, "main_record_seconds": 127.255, "model_responses": 11, "outer_calls": 10, "nested_operations": null, "stop_reason": "stop", "source_sha256": "23479d37cff80bd8ae7f22efdaa9ba5c7cd9825c65c89b9fcad3bb6164b5ad8b", "uncached_input": 48114, "cached_input": 396672, "cache_write": 0, "output": 2849, "estimated_usd": 1.020262, "total_tokens": 447635, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/G3-codex" }, { "run": "H1-sunnycode", "task": "H", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T06:25:15.410Z", "final_record_at": "2026-10-14T06:29:14.607Z", "submission_to_final_seconds": 239.687, "main_record_seconds": 239.197, "model_responses": 21, "outer_calls": 26, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "1734e131441293406401f2164cf7ebe933876af4668e8f1d9d8f48e30cc8f95d", "uncached_input": 21510, "cached_input": 252160, "cache_write": 0, "output": 4481, "estimated_usd": 0.69131, "total_tokens": 278151, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/H1-sunnycode" }, { "run": "H1-sunnycode", "task": "H", "repeat": 1, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T18:17:05.481Z", "final_record_at": "2026-10-13T18:23:49.580Z", "submission_to_final_seconds": 404.502, "main_record_seconds": 404.099, "model_responses": 12, "outer_calls": 11, "nested_operations": 37, "stop_reason": "stop", "source_sha256": "0d53d478f677d3f01b9edc0d3ca922eb9acbd99623f9711b9cde894bc90b57a1", "uncached_input": 25360, "cached_input": 162432, "cache_write": 0, "output": 5936, "estimated_usd": 0.712832, "total_tokens": 193728, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/H1-sunnycode" }, { "run": "H1-codex", "task": "H", "repeat": 1, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-13T18:24:14.797Z", "final_record_at": "2026-10-13T18:26:25.793Z", "submission_to_final_seconds": 135.201, "main_record_seconds": 130.996, "model_responses": 10, "outer_calls": 9, "nested_operations": null, "stop_reason": "stop", "source_sha256": "415071e118fd81e27bd1f77fe9136f671acf952d0e2f0211d9898f5779fb7bdf", "uncached_input": 50422, "cached_input": 371584, "cache_write": 0, "output": 4288, "estimated_usd": 1.090204, "total_tokens": 426294, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/H1-codex" }, { "run": "H2-sunnycode", "task": "H", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T06:30:58.794Z", "final_record_at": "2026-10-14T06:34:17.393Z", "submission_to_final_seconds": 199.28, "main_record_seconds": 198.599, "model_responses": 14, "outer_calls": 31, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "7a823c89d1b8a1fe8f2717cfaab1b2e4d7b498925175f5dd614b018c9d817a7f", "uncached_input": 28860, "cached_input": 210688, "cache_write": 0, "output": 5421, "estimated_usd": 0.770338, "total_tokens": 244969, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/H2-sunnycode" }, { "run": "H2-sunnycode", "task": "H", "repeat": 2, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:08:02.382Z", "final_record_at": "2026-10-14T03:11:39.373Z", "submission_to_final_seconds": 217.524, "main_record_seconds": 216.991, "model_responses": 13, "outer_calls": 12, "nested_operations": 38, "stop_reason": "stop", "source_sha256": "f8adf973a10d49f977610932962c7b54ae9ce9762700aacec32683648c44ab16", "uncached_input": 32639, "cached_input": 231936, "cache_write": 0, "output": 6726, "estimated_usd": 0.8946259999999999, "total_tokens": 271301, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/H2-sunnycode" }, { "run": "H2-codex", "task": "H", "repeat": 2, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:12:16.753Z", "final_record_at": "2026-10-14T03:14:59.267Z", "submission_to_final_seconds": 167.922, "main_record_seconds": 162.514, "model_responses": 11, "outer_calls": 10, "nested_operations": null, "stop_reason": "stop", "source_sha256": "e69f9bd5cc4692b53344c3815cc998bdc10e0dd7312798e8bba3e4be94abd819", "uncached_input": 29308, "cached_input": 446592, "cache_write": 0, "output": 4509, "estimated_usd": 0.965122, "total_tokens": 480409, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/H2-codex" }, { "run": "H3-sunnycode", "task": "H", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T08:47:06.702Z", "final_record_at": "2026-10-14T08:49:58.308Z", "submission_to_final_seconds": 172.132, "main_record_seconds": 171.606, "model_responses": 14, "outer_calls": 27, "nested_operations": 0, "stop_reason": "stop", "source_sha256": "3016fe912122d7f696de81a608a988c6cc6be191b8f445665c4c429d7f271025", "uncached_input": 25207, "cached_input": 155648, "cache_write": 0, "output": 4368, "estimated_usd": 0.626118, "total_tokens": 185223, "arm": "base", "supplemental": false, "passed": true, "evidence": "three-arm-completion/runs/H3-sunnycode" }, { "run": "H3-sunnycode", "task": "H", "repeat": 3, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:45:38.303Z", "final_record_at": "2026-10-14T03:49:10.748Z", "submission_to_final_seconds": 213.124, "main_record_seconds": 212.445, "model_responses": 13, "outer_calls": 12, "nested_operations": 35, "stop_reason": "stop", "source_sha256": "f39b195bcb5b390673fd03a6d1b8e55a7f5d1814a0967de2f37b31582845166c", "uncached_input": 36234, "cached_input": 200960, "cache_write": 0, "output": 6646, "estimated_usd": 0.8956, "total_tokens": 243840, "arm": "codemode", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/H3-sunnycode" }, { "run": "H3-codex", "task": "H", "repeat": 3, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T03:49:39.120Z", "final_record_at": "2026-10-14T03:51:43.882Z", "submission_to_final_seconds": 129.283, "main_record_seconds": 124.762, "model_responses": 10, "outer_calls": 9, "nested_operations": null, "stop_reason": "stop", "source_sha256": "63199502c4e745c0340972aa4e1d62fd8f9a19039ab09d9fe02ce40d373ab32c", "uncached_input": 29810, "cached_input": 377216, "cache_write": 0, "output": 3702, "estimated_usd": 0.860416, "total_tokens": 410728, "arm": "codex", "supplemental": false, "passed": true, "evidence": "cross-repo-v1/runs/H3-codex" } ], "summaries": [ { "task": "A", "title": "addAllSettled()", "arm": "base", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 299.00666666666666, "median": 311.06, "min": 237.104, "max": 348.856 }, "metrics": { "submission_to_final_seconds": { "mean": 299.00666666666666, "median": 311.06, "min": 237.104, "max": 348.856 }, "estimated_usd": { "mean": 0.7775506666666667, "median": 0.759844, "min": 0.742214, "max": 0.830594 }, "uncached_input": { "mean": 35083, "median": 33363, "min": 33331, "max": 38555 }, "cached_input": { "mean": 212437.33333333334, "median": 203904, "min": 203264, "max": 230144 }, "output": { "mean": 4285.666666666667, "median": 4298, "min": 4100, "max": 4459 }, "total_tokens": { "mean": 251806, "median": 241335, "min": 241086, "max": 272997 }, "model_responses": { "mean": 19.333333333333332, "median": 19, "min": 19, "max": 20 }, "outer_calls": { "mean": 22, "median": 22, "min": 21, "max": 23 } } }, { "task": "A", "title": "addAllSettled()", "arm": "codemode", "planned": 3, "completed": 3, "graded": 3, "passed": 2, "normal_final_responses": 3, "successful_completion_seconds": { "n": 2, "mean": 200.624, "median": 200.624, "min": 191.732, "max": 209.516 }, "metrics": { "submission_to_final_seconds": { "mean": 218.13533333333334, "median": 209.516, "min": 191.732, "max": 253.158 }, "estimated_usd": { "mean": 0.46577666666666667, "median": 0.44977, "min": 0.4455, "max": 0.50206 }, "uncached_input": { "mean": 21317, "median": 18026, "min": 16626, "max": 29299 }, "cached_input": { "mean": 67840, "median": 71040, "min": 40320, "max": 92160 }, "output": { "mean": 3695.3333333333335, "median": 3827, "min": 3375, "max": 3884 }, "total_tokens": { "mean": 92852.33333333333, "median": 92950, "min": 72994, "max": 112613 }, "model_responses": { "mean": 8.333333333333334, "median": 8, "min": 7, "max": 10 }, "outer_calls": { "mean": 7.333333333333333, "median": 7, "min": 6, "max": 9 } } }, { "task": "A", "title": "addAllSettled()", "arm": "codex", "planned": 3, "completed": 3, "graded": 3, "passed": 1, "normal_final_responses": 3, "successful_completion_seconds": { "n": 1, "mean": 243.327, "median": 243.327, "min": 243.327, "max": 243.327 }, "metrics": { "submission_to_final_seconds": { "mean": 225.90766666666667, "median": 236.861, "min": 197.535, "max": 243.327 }, "estimated_usd": { "mean": 1.7196006666666666, "median": 1.87599, "min": 1.259204, "max": 2.023608 }, "uncached_input": { "mean": 57646.666666666664, "median": 60868, "min": 47064, "max": 65008 }, "cached_input": { "mean": 905984, "median": 1013760, "min": 590464, "max": 1113728 }, "output": { "mean": 4743, "median": 5071, "min": 3962, "max": 5196 }, "total_tokens": { "mean": 968373.6666666666, "median": 1079699, "min": 641490, "max": 1183932 }, "model_responses": { "mean": 20.666666666666668, "median": 23, "min": 16, "max": 23 }, "outer_calls": { "mean": 19.666666666666668, "median": 22, "min": 15, "max": 22 } } }, { "task": "B", "title": "onEmpty() notification timing", "arm": "base", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 272.257, "median": 297.484, "min": 210.905, "max": 308.382 }, "metrics": { "submission_to_final_seconds": { "mean": 272.257, "median": 297.484, "min": 210.905, "max": 308.382 }, "estimated_usd": { "mean": 0.8396399999999999, "median": 0.790172, "min": 0.676906, "max": 1.051842 }, "uncached_input": { "mean": 31075.666666666668, "median": 30464, "min": 24251, "max": 38512 }, "cached_input": { "mean": 264533.3333333333, "median": 215296, "min": 196352, "max": 381952 }, "output": { "mean": 5287, "median": 4382, "min": 4174, "max": 7305 }, "total_tokens": { "mean": 300896, "median": 243929, "min": 239038, "max": 419721 }, "model_responses": { "mean": 18, "median": 17, "min": 15, "max": 22 }, "outer_calls": { "mean": 18.333333333333332, "median": 16, "min": 16, "max": 23 } } }, { "task": "B", "title": "onEmpty() notification timing", "arm": "codemode", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 197.84233333333333, "median": 187.192, "min": 162.191, "max": 244.144 }, "metrics": { "submission_to_final_seconds": { "mean": 197.84233333333333, "median": 187.192, "min": 162.191, "max": 244.144 }, "estimated_usd": { "mean": 0.6774086666666667, "median": 0.597504, "min": 0.557362, "max": 0.87736 }, "uncached_input": { "mean": 33563.666666666664, "median": 27230, "min": 26542, "max": 46919 }, "cached_input": { "mean": 123605.33333333333, "median": 114304, "min": 113792, "max": 142720 }, "output": { "mean": 4363.333333333333, "median": 4218, "min": 3563, "max": 5309 }, "total_tokens": { "mean": 161532.33333333334, "median": 145752, "min": 143897, "max": 194948 }, "model_responses": { "mean": 9.666666666666666, "median": 9, "min": 9, "max": 11 }, "outer_calls": { "mean": 8.666666666666666, "median": 8, "min": 8, "max": 10 } } }, { "task": "B", "title": "onEmpty() notification timing", "arm": "codex", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 253.47333333333333, "median": 220.993, "min": 217.103, "max": 322.324 }, "metrics": { "submission_to_final_seconds": { "mean": 253.47333333333333, "median": 220.993, "min": 217.103, "max": 322.324 }, "estimated_usd": { "mean": 1.6616226666666667, "median": 1.685324, "min": 1.57946, "max": 1.720084 }, "uncached_input": { "mean": 52070.333333333336, "median": 57715, "min": 39170, "max": 59326 }, "cached_input": { "mean": 832469.3333333334, "median": 851584, "min": 688000, "max": 957824 }, "output": { "mean": 6169, "median": 5964, "min": 5827, "max": 6716 }, "total_tokens": { "mean": 890708.6666666666, "median": 915126, "min": 753290, "max": 1003710 }, "model_responses": { "mean": 17.666666666666668, "median": 19, "min": 15, "max": 19 }, "outer_calls": { "mean": 16.666666666666668, "median": 18, "min": 14, "max": 18 } } }, { "task": "C", "title": "Streaming task results", "arm": "base", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 602.9780000000001, "median": 499.68, "min": 433.221, "max": 876.033 }, "metrics": { "submission_to_final_seconds": { "mean": 602.9780000000001, "median": 499.68, "min": 433.221, "max": 876.033 }, "estimated_usd": { "mean": 1.891912, "median": 1.945628, "min": 1.608222, "max": 2.121886 }, "uncached_input": { "mean": 55952, "median": 62547, "min": 37443, "max": 67866 }, "cached_input": { "mean": 731392, "median": 650368, "min": 509952, "max": 1033856 }, "output": { "mean": 12020, "median": 12332, "min": 9456, "max": 14272 }, "total_tokens": { "mean": 799364, "median": 730566, "min": 581955, "max": 1085571 }, "model_responses": { "mean": 37.666666666666664, "median": 33, "min": 29, "max": 51 }, "outer_calls": { "mean": 41, "median": 39, "min": 31, "max": 53 } } }, { "task": "C", "title": "Streaming task results", "arm": "codemode", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 440.5323333333333, "median": 440.497, "min": 418.146, "max": 462.954 }, "metrics": { "submission_to_final_seconds": { "mean": 440.5323333333333, "median": 440.497, "min": 418.146, "max": 462.954 }, "estimated_usd": { "mean": 1.246882, "median": 1.30203, "min": 1.119882, "max": 1.318734 }, "uncached_input": { "mean": 45300.333333333336, "median": 43623, "min": 33232, "max": 59046 }, "cached_input": { "mean": 298112, "median": 317312, "min": 229120, "max": 347904 }, "output": { "mean": 9915.333333333334, "median": 9649, "min": 9405, "max": 10692 }, "total_tokens": { "mean": 353327.6666666667, "median": 359949, "min": 297815, "max": 402219 }, "model_responses": { "mean": 15.333333333333334, "median": 16, "min": 14, "max": 16 }, "outer_calls": { "mean": 14.333333333333334, "median": 15, "min": 13, "max": 15 } } }, { "task": "C", "title": "Streaming task results", "arm": "codex", "planned": 3, "completed": 3, "graded": 3, "passed": 2, "normal_final_responses": 3, "successful_completion_seconds": { "n": 2, "mean": 500.874, "median": 500.874, "min": 465.826, "max": 535.922 }, "metrics": { "submission_to_final_seconds": { "mean": 530.1523333333333, "median": 535.922, "min": 465.826, "max": 588.709 }, "estimated_usd": { "mean": 2.7951606666666664, "median": 2.741916, "min": 2.659548, "max": 2.984018 }, "uncached_input": { "mean": 77057.66666666667, "median": 77760, "min": 73010, "max": 80403 }, "cached_input": { "mean": 1362517.3333333333, "median": 1324416, "min": 1205248, "max": 1557888 }, "output": { "mean": 13241.333333333334, "median": 13534, "min": 12442, "max": 13748 }, "total_tokens": { "mean": 1452816.3333333333, "median": 1411174, "min": 1296542, "max": 1650733 }, "model_responses": { "mean": 24.333333333333332, "median": 24, "min": 22, "max": 27 }, "outer_calls": { "mean": 23.333333333333332, "median": 23, "min": 21, "max": 26 } } }, { "task": "D", "title": "Graceful queue shutdown", "arm": "base", "planned": 3, "completed": 3, "graded": 3, "passed": 2, "normal_final_responses": 2, "successful_completion_seconds": { "n": 2, "mean": 504.288, "median": 504.288, "min": 441.368, "max": 567.208 }, "metrics": { "submission_to_final_seconds": { "mean": 395.628, "median": 441.368, "min": 178.308, "max": 567.208 }, "estimated_usd": { "mean": 0.8138346666666667, "median": 0.944968, "min": 0.396388, "max": 1.100148 }, "uncached_input": { "mean": 31359.333333333332, "median": 27733, "min": 27348, "max": 38997 }, "cached_input": { "mean": 238208, "median": 332928, "min": 30208, "max": 351488 }, "output": { "mean": 5240.666666666667, "median": 6400, "min": 1777, "max": 7545 }, "total_tokens": { "mean": 274808, "median": 379470, "min": 59718, "max": 385236 }, "model_responses": { "mean": 17.333333333333332, "median": 21, "min": 10, "max": 21 }, "outer_calls": { "mean": 20.333333333333332, "median": 23, "min": 13, "max": 25 } } }, { "task": "D", "title": "Graceful queue shutdown", "arm": "codemode", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 438.05133333333333, "median": 316.999, "min": 254.44, "max": 742.715 }, "metrics": { "submission_to_final_seconds": { "mean": 438.05133333333333, "median": 316.999, "min": 254.44, "max": 742.715 }, "estimated_usd": { "mean": 0.879404, "median": 0.881732, "min": 0.85856, "max": 0.8979199999999999 }, "uncached_input": { "mean": 37921.666666666664, "median": 39785, "min": 30443, "max": 43537 }, "cached_input": { "mean": 183637.33333333334, "median": 175232, "min": 150400, "max": 225280 }, "output": { "mean": 6331, "median": 6243, "min": 6173, "max": 6577 }, "total_tokens": { "mean": 227890, "median": 221190, "min": 200180, "max": 262300 }, "model_responses": { "mean": 13, "median": 12, "min": 11, "max": 16 }, "outer_calls": { "mean": 11.333333333333334, "median": 11, "min": 10, "max": 13 } } }, { "task": "D", "title": "Graceful queue shutdown", "arm": "codex", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 298.407, "median": 294.742, "min": 271.907, "max": 328.572 }, "metrics": { "submission_to_final_seconds": { "mean": 298.407, "median": 294.742, "min": 271.907, "max": 328.572 }, "estimated_usd": { "mean": 1.8871993333333335, "median": 1.959204, "min": 1.7190860000000001, "max": 1.983308 }, "uncached_input": { "mean": 55105.333333333336, "median": 62112, "min": 39383, "max": 63821 }, "cached_input": { "mean": 908629.3333333334, "median": 909184, "min": 878848, "max": 937856 }, "output": { "mean": 8550.333333333334, "median": 8578, "min": 7748, "max": 9325 }, "total_tokens": { "mean": 972285, "median": 979874, "min": 951994, "max": 984987 }, "model_responses": { "mean": 18.666666666666668, "median": 19, "min": 18, "max": 19 }, "outer_calls": { "mean": 17.666666666666668, "median": 18, "min": 17, "max": 18 } } }, { "task": "E", "title": "Environment-variable hints", "arm": "base", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 103.54566666666666, "median": 103.249, "min": 60.067, "max": 147.321 }, "metrics": { "submission_to_final_seconds": { "mean": 103.54566666666666, "median": 103.249, "min": 60.067, "max": 147.321 }, "estimated_usd": { "mean": 0.275458, "median": 0.27699, "min": 0.21857000000000001, "max": 0.330814 }, "uncached_input": { "mean": 13199.666666666666, "median": 11310, "min": 8799, "max": 19490 }, "cached_input": { "mean": 69461.33333333333, "median": 65280, "min": 62464, "max": 80640 }, "output": { "mean": 1480, "median": 1469, "min": 1306, "max": 1665 }, "total_tokens": { "mean": 84141, "median": 83423, "min": 75385, "max": 93615 }, "model_responses": { "mean": 9.666666666666666, "median": 9, "min": 8, "max": 12 }, "outer_calls": { "mean": 9.333333333333334, "median": 9, "min": 8, "max": 11 } } }, { "task": "E", "title": "Environment-variable hints", "arm": "codemode", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 63.47933333333334, "median": 67.431, "min": 53.714, "max": 69.293 }, "metrics": { "submission_to_final_seconds": { "mean": 63.47933333333334, "median": 67.431, "min": 53.714, "max": 69.293 }, "estimated_usd": { "mean": 0.27863933333333335, "median": 0.268194, "min": 0.251734, "max": 0.31599 }, "uncached_input": { "mean": 15882.666666666666, "median": 15551, "min": 11368, "max": 20729 }, "cached_input": { "mean": 38229.333333333336, "median": 41984, "min": 25600, "max": 47104 }, "output": { "mean": 1631.6666666666667, "median": 1662, "min": 1414, "max": 1819 }, "total_tokens": { "mean": 55743.666666666664, "median": 58949, "min": 47991, "max": 60291 }, "model_responses": { "mean": 7, "median": 7, "min": 6, "max": 8 }, "outer_calls": { "mean": 6, "median": 6, "min": 5, "max": 7 } } }, { "task": "E", "title": "Environment-variable hints", "arm": "codex", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 72.78333333333333, "median": 70.068, "min": 64.094, "max": 84.188 }, "metrics": { "submission_to_final_seconds": { "mean": 72.78333333333333, "median": 70.068, "min": 64.094, "max": 84.188 }, "estimated_usd": { "mean": 0.648938, "median": 0.593252, "min": 0.56711, "max": 0.786452 }, "uncached_input": { "mean": 30366.666666666668, "median": 25250, "min": 22362, "max": 43488 }, "cached_input": { "mean": 259754.66666666666, "median": 255872, "min": 239360, "max": 284032 }, "output": { "mean": 1710.3333333333333, "median": 1712, "min": 1505, "max": 1914 }, "total_tokens": { "mean": 291831.6666666667, "median": 301274, "min": 266115, "max": 308106 }, "model_responses": { "mean": 7.666666666666667, "median": 8, "min": 7, "max": 8 }, "outer_calls": { "mean": 6.666666666666667, "median": 7, "min": 6, "max": 7 } } }, { "task": "F", "title": "Canonical choice aliases", "arm": "base", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 185.86766666666668, "median": 159.311, "min": 143.84, "max": 254.452 }, "metrics": { "submission_to_final_seconds": { "mean": 185.86766666666668, "median": 159.311, "min": 143.84, "max": 254.452 }, "estimated_usd": { "mean": 0.5037466666666667, "median": 0.465668, "min": 0.452834, "max": 0.592738 }, "uncached_input": { "mean": 20687.333333333332, "median": 15932, "min": 15890, "max": 30240 }, "cached_input": { "mean": 126506.66666666667, "median": 125568, "min": 121088, "max": 132864 }, "output": { "mean": 3407.3333333333335, "median": 3385, "min": 3213, "max": 3624 }, "total_tokens": { "mean": 150601.33333333334, "median": 152009, "min": 145082, "max": 154713 }, "model_responses": { "mean": 13.666666666666666, "median": 14, "min": 13, "max": 14 }, "outer_calls": { "mean": 14.666666666666666, "median": 15, "min": 13, "max": 16 } } }, { "task": "F", "title": "Canonical choice aliases", "arm": "codemode", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 116.522, "median": 107.311, "min": 105.283, "max": 136.972 }, "metrics": { "submission_to_final_seconds": { "mean": 116.522, "median": 107.311, "min": 105.283, "max": 136.972 }, "estimated_usd": { "mean": 0.5042139999999999, "median": 0.487558, "min": 0.45184399999999997, "max": 0.57324 }, "uncached_input": { "mean": 26353, "median": 26087, "min": 25619, "max": 27353 }, "cached_input": { "mean": 67584, "median": 73088, "min": 40704, "max": 88960 }, "output": { "mean": 3462, "median": 3099, "min": 3072, "max": 4215 }, "total_tokens": { "mean": 97399, "median": 102247, "min": 69422, "max": 120528 }, "model_responses": { "mean": 8.666666666666666, "median": 9, "min": 8, "max": 9 }, "outer_calls": { "mean": 7.666666666666667, "median": 8, "min": 7, "max": 8 } } }, { "task": "F", "title": "Canonical choice aliases", "arm": "codex", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 120.82566666666666, "median": 118.958, "min": 116.883, "max": 126.636 }, "metrics": { "submission_to_final_seconds": { "mean": 120.82566666666666, "median": 118.958, "min": 116.883, "max": 126.636 }, "estimated_usd": { "mean": 0.9021966666666666, "median": 0.9280200000000001, "min": 0.833678, "max": 0.944892 }, "uncached_input": { "mean": 39901.666666666664, "median": 46029, "min": 25518, "max": 48158 }, "cached_input": { "mean": 320213.3333333333, "median": 308352, "min": 279040, "max": 373248 }, "output": { "mean": 3659.3333333333335, "median": 3525, "min": 3348, "max": 4105 }, "total_tokens": { "mean": 363774.3333333333, "median": 357906, "min": 330546, "max": 402871 }, "model_responses": { "mean": 9, "median": 9, "min": 8, "max": 10 }, "outer_calls": { "mean": 8, "median": 8, "min": 7, "max": 9 } } }, { "task": "G", "title": "Reference-definition boundaries", "arm": "base", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 314.2006666666667, "median": 227.895, "min": 126.909, "max": 587.798 }, "metrics": { "submission_to_final_seconds": { "mean": 314.2006666666667, "median": 227.895, "min": 126.909, "max": 587.798 }, "estimated_usd": { "mean": 0.481078, "median": 0.525088, "min": 0.380428, "max": 0.537718 }, "uncached_input": { "mean": 21218, "median": 19931, "min": 15936, "max": 27787 }, "cached_input": { "mean": 126848, "median": 117248, "min": 99968, "max": 163328 }, "output": { "mean": 2841, "median": 2852, "min": 2422, "max": 3249 }, "total_tokens": { "mean": 150907, "median": 147887, "min": 118326, "max": 186508 }, "model_responses": { "mean": 20, "median": 16, "min": 11, "max": 33 }, "outer_calls": { "mean": 15, "median": 16, "min": 12, "max": 17 } } }, { "task": "G", "title": "Reference-definition boundaries", "arm": "codemode", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 104.91666666666667, "median": 97.471, "min": 91.469, "max": 125.81 }, "metrics": { "submission_to_final_seconds": { "mean": 104.91666666666667, "median": 97.471, "min": 91.469, "max": 125.81 }, "estimated_usd": { "mean": 0.46610866666666667, "median": 0.462454, "min": 0.399152, "max": 0.53672 }, "uncached_input": { "mean": 25328.666666666668, "median": 26094, "min": 20465, "max": 29427 }, "cached_input": { "mean": 71338.66666666667, "median": 75264, "min": 61952, "max": 76800 }, "output": { "mean": 2829.6666666666665, "median": 2651, "min": 2525, "max": 3313 }, "total_tokens": { "mean": 99497, "median": 103883, "min": 85068, "max": 109540 }, "model_responses": { "mean": 8.333333333333334, "median": 8, "min": 8, "max": 9 }, "outer_calls": { "mean": 7.333333333333333, "median": 7, "min": 7, "max": 8 } } }, { "task": "G", "title": "Reference-definition boundaries", "arm": "codex", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 135.19666666666666, "median": 134.384, "min": 134.349, "max": 136.857 }, "metrics": { "submission_to_final_seconds": { "mean": 135.19666666666666, "median": 134.384, "min": 134.349, "max": 136.857 }, "estimated_usd": { "mean": 0.935534, "median": 0.994392, "min": 0.791948, "max": 1.020262 }, "uncached_input": { "mean": 39930.666666666664, "median": 46924, "min": 24754, "max": 48114 }, "cached_input": { "mean": 379477.3333333333, "median": 385408, "min": 356352, "max": 396672 }, "output": { "mean": 3135, "median": 3180, "min": 2849, "max": 3376 }, "total_tokens": { "mean": 422543, "median": 413342, "min": 406652, "max": 447635 }, "model_responses": { "mean": 10.333333333333334, "median": 10, "min": 10, "max": 11 }, "outer_calls": { "mean": 9.333333333333334, "median": 9, "min": 9, "max": 10 } } }, { "task": "H", "title": "Opt-in stable heading IDs", "arm": "base", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 203.69966666666667, "median": 199.28, "min": 172.132, "max": 239.687 }, "metrics": { "submission_to_final_seconds": { "mean": 203.69966666666667, "median": 199.28, "min": 172.132, "max": 239.687 }, "estimated_usd": { "mean": 0.6959219999999999, "median": 0.69131, "min": 0.626118, "max": 0.770338 }, "uncached_input": { "mean": 25192.333333333332, "median": 25207, "min": 21510, "max": 28860 }, "cached_input": { "mean": 206165.33333333334, "median": 210688, "min": 155648, "max": 252160 }, "output": { "mean": 4756.666666666667, "median": 4481, "min": 4368, "max": 5421 }, "total_tokens": { "mean": 236114.33333333334, "median": 244969, "min": 185223, "max": 278151 }, "model_responses": { "mean": 16.333333333333332, "median": 14, "min": 14, "max": 21 }, "outer_calls": { "mean": 28, "median": 27, "min": 26, "max": 31 } } }, { "task": "H", "title": "Opt-in stable heading IDs", "arm": "codemode", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 278.3833333333333, "median": 217.524, "min": 213.124, "max": 404.502 }, "metrics": { "submission_to_final_seconds": { "mean": 278.3833333333333, "median": 217.524, "min": 213.124, "max": 404.502 }, "estimated_usd": { "mean": 0.8343526666666666, "median": 0.8946259999999999, "min": 0.712832, "max": 0.8956 }, "uncached_input": { "mean": 31411, "median": 32639, "min": 25360, "max": 36234 }, "cached_input": { "mean": 198442.66666666666, "median": 200960, "min": 162432, "max": 231936 }, "output": { "mean": 6436, "median": 6646, "min": 5936, "max": 6726 }, "total_tokens": { "mean": 236289.66666666666, "median": 243840, "min": 193728, "max": 271301 }, "model_responses": { "mean": 12.666666666666666, "median": 13, "min": 12, "max": 13 }, "outer_calls": { "mean": 11.666666666666666, "median": 12, "min": 11, "max": 12 } } }, { "task": "H", "title": "Opt-in stable heading IDs", "arm": "codex", "planned": 3, "completed": 3, "graded": 3, "passed": 3, "normal_final_responses": 3, "successful_completion_seconds": { "n": 3, "mean": 144.13533333333334, "median": 135.201, "min": 129.283, "max": 167.922 }, "metrics": { "submission_to_final_seconds": { "mean": 144.13533333333334, "median": 135.201, "min": 129.283, "max": 167.922 }, "estimated_usd": { "mean": 0.971914, "median": 0.965122, "min": 0.860416, "max": 1.090204 }, "uncached_input": { "mean": 36513.333333333336, "median": 29810, "min": 29308, "max": 50422 }, "cached_input": { "mean": 398464, "median": 377216, "min": 371584, "max": 446592 }, "output": { "mean": 4166.333333333333, "median": 4288, "min": 3702, "max": 4509 }, "total_tokens": { "mean": 439143.6666666667, "median": 426294, "min": 410728, "max": 480409 }, "model_responses": { "mean": 10.333333333333334, "median": 10, "min": 10, "max": 11 }, "outer_calls": { "mean": 9.333333333333334, "median": 9, "min": 9, "max": 10 } } } ], "supplemental": [ { "run": "H4-sunnycode", "task": "H", "repeat": 4, "product": "sunnycode", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T05:05:57.638Z", "final_record_at": "2026-10-14T05:09:19.852Z", "submission_to_final_seconds": 203.025, "main_record_seconds": 202.214, "model_responses": 12, "outer_calls": 11, "nested_operations": 37, "stop_reason": "stop", "source_sha256": "42f2be5b863618c82f024c53f08915af1519472f77a7d5cde328ae4b75aa7231", "uncached_input": 37614, "cached_input": 211328, "cache_write": 0, "output": 7451, "estimated_usd": 0.960018, "total_tokens": 256393, "arm": "codemode", "supplemental": true, "passed": true, "evidence": "three-arm-completion/runs/H4-sunnycode" }, { "run": "H4-codex", "task": "H", "repeat": 4, "product": "codex", "model": "gpt-6-astra", "reasoning": "medium", "task_input_at": "2026-10-14T05:10:01.668Z", "final_record_at": "2026-10-14T05:12:47.698Z", "submission_to_final_seconds": 171.737, "main_record_seconds": 166.03, "model_responses": 14, "outer_calls": 13, "nested_operations": null, "stop_reason": "stop", "source_sha256": "c6d5e2ebc37d4a05ad8a0a0e1a8e014082bbf3948bf5bbe380ef5ac220c60912", "uncached_input": 53729, "cached_input": 560896, "cache_write": 0, "output": 4917, "estimated_usd": 1.344036, "total_tokens": 619542, "arm": "codex", "supplemental": true, "passed": true, "evidence": "three-arm-completion/runs/H4-codex" } ], "publication_note": "Local source_log paths are omitted; source hashes, run identifiers, measurements and unsuccessful outcomes are retained." }